7a9182af9bfe7401706b1409f9b27ddb95ccbf10
[ffmpeg.git] / libavcodec / hevcdec.c
1 /*
2  * HEVC video decoder
3  *
4  * Copyright (C) 2012 - 2013 Guillaume Martres
5  * Copyright (C) 2012 - 2013 Mickael Raulet
6  * Copyright (C) 2012 - 2013 Gildas Cocherel
7  * Copyright (C) 2012 - 2013 Wassim Hamidouche
8  *
9  * This file is part of Libav.
10  *
11  * Libav is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU Lesser General Public
13  * License as published by the Free Software Foundation; either
14  * version 2.1 of the License, or (at your option) any later version.
15  *
16  * Libav is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19  * Lesser General Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser General Public
22  * License along with Libav; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24  */
25
26 #include "libavutil/attributes.h"
27 #include "libavutil/common.h"
28 #include "libavutil/display.h"
29 #include "libavutil/internal.h"
30 #include "libavutil/md5.h"
31 #include "libavutil/opt.h"
32 #include "libavutil/pixdesc.h"
33 #include "libavutil/stereo3d.h"
34
35 #include "bswapdsp.h"
36 #include "bytestream.h"
37 #include "cabac_functions.h"
38 #include "golomb_legacy.h"
39 #include "hevc.h"
40 #include "hevc_data.h"
41 #include "hevcdec.h"
42 #include "profiles.h"
43
44 const uint8_t ff_hevc_qpel_extra_before[4] = { 0, 3, 3, 3 };
45 const uint8_t ff_hevc_qpel_extra_after[4]  = { 0, 4, 4, 4 };
46 const uint8_t ff_hevc_qpel_extra[4]        = { 0, 7, 7, 7 };
47
48 static const uint8_t scan_1x1[1] = { 0 };
49
50 static const uint8_t horiz_scan2x2_x[4] = { 0, 1, 0, 1 };
51
52 static const uint8_t horiz_scan2x2_y[4] = { 0, 0, 1, 1 };
53
54 static const uint8_t horiz_scan4x4_x[16] = {
55     0, 1, 2, 3,
56     0, 1, 2, 3,
57     0, 1, 2, 3,
58     0, 1, 2, 3,
59 };
60
61 static const uint8_t horiz_scan4x4_y[16] = {
62     0, 0, 0, 0,
63     1, 1, 1, 1,
64     2, 2, 2, 2,
65     3, 3, 3, 3,
66 };
67
68 static const uint8_t horiz_scan8x8_inv[8][8] = {
69     {  0,  1,  2,  3, 16, 17, 18, 19, },
70     {  4,  5,  6,  7, 20, 21, 22, 23, },
71     {  8,  9, 10, 11, 24, 25, 26, 27, },
72     { 12, 13, 14, 15, 28, 29, 30, 31, },
73     { 32, 33, 34, 35, 48, 49, 50, 51, },
74     { 36, 37, 38, 39, 52, 53, 54, 55, },
75     { 40, 41, 42, 43, 56, 57, 58, 59, },
76     { 44, 45, 46, 47, 60, 61, 62, 63, },
77 };
78
79 static const uint8_t diag_scan2x2_x[4] = { 0, 0, 1, 1 };
80
81 static const uint8_t diag_scan2x2_y[4] = { 0, 1, 0, 1 };
82
83 static const uint8_t diag_scan2x2_inv[2][2] = {
84     { 0, 2, },
85     { 1, 3, },
86 };
87
88 static const uint8_t diag_scan4x4_inv[4][4] = {
89     { 0,  2,  5,  9, },
90     { 1,  4,  8, 12, },
91     { 3,  7, 11, 14, },
92     { 6, 10, 13, 15, },
93 };
94
95 static const uint8_t diag_scan8x8_inv[8][8] = {
96     {  0,  2,  5,  9, 14, 20, 27, 35, },
97     {  1,  4,  8, 13, 19, 26, 34, 42, },
98     {  3,  7, 12, 18, 25, 33, 41, 48, },
99     {  6, 11, 17, 24, 32, 40, 47, 53, },
100     { 10, 16, 23, 31, 39, 46, 52, 57, },
101     { 15, 22, 30, 38, 45, 51, 56, 60, },
102     { 21, 29, 37, 44, 50, 55, 59, 62, },
103     { 28, 36, 43, 49, 54, 58, 61, 63, },
104 };
105
106 /**
107  * NOTE: Each function hls_foo correspond to the function foo in the
108  * specification (HLS stands for High Level Syntax).
109  */
110
111 /**
112  * Section 5.7
113  */
114
115 /* free everything allocated  by pic_arrays_init() */
116 static void pic_arrays_free(HEVCContext *s)
117 {
118     av_freep(&s->sao);
119     av_freep(&s->deblock);
120
121     av_freep(&s->skip_flag);
122     av_freep(&s->tab_ct_depth);
123
124     av_freep(&s->tab_ipm);
125     av_freep(&s->cbf_luma);
126     av_freep(&s->is_pcm);
127
128     av_freep(&s->qp_y_tab);
129     av_freep(&s->tab_slice_address);
130     av_freep(&s->filter_slice_edges);
131
132     av_freep(&s->horizontal_bs);
133     av_freep(&s->vertical_bs);
134
135     av_buffer_pool_uninit(&s->tab_mvf_pool);
136     av_buffer_pool_uninit(&s->rpl_tab_pool);
137 }
138
139 /* allocate arrays that depend on frame dimensions */
140 static int pic_arrays_init(HEVCContext *s, const HEVCSPS *sps)
141 {
142     int log2_min_cb_size = sps->log2_min_cb_size;
143     int width            = sps->width;
144     int height           = sps->height;
145     int pic_size_in_ctb  = ((width  >> log2_min_cb_size) + 1) *
146                            ((height >> log2_min_cb_size) + 1);
147     int ctb_count        = sps->ctb_width * sps->ctb_height;
148     int min_pu_size      = sps->min_pu_width * sps->min_pu_height;
149
150     s->bs_width  = width  >> 3;
151     s->bs_height = height >> 3;
152
153     s->sao           = av_mallocz_array(ctb_count, sizeof(*s->sao));
154     s->deblock       = av_mallocz_array(ctb_count, sizeof(*s->deblock));
155     if (!s->sao || !s->deblock)
156         goto fail;
157
158     s->skip_flag    = av_malloc(pic_size_in_ctb);
159     s->tab_ct_depth = av_malloc(sps->min_cb_height * sps->min_cb_width);
160     if (!s->skip_flag || !s->tab_ct_depth)
161         goto fail;
162
163     s->cbf_luma = av_malloc(sps->min_tb_width * sps->min_tb_height);
164     s->tab_ipm  = av_mallocz(min_pu_size);
165     s->is_pcm   = av_malloc(min_pu_size);
166     if (!s->tab_ipm || !s->cbf_luma || !s->is_pcm)
167         goto fail;
168
169     s->filter_slice_edges = av_malloc(ctb_count);
170     s->tab_slice_address  = av_malloc(pic_size_in_ctb *
171                                       sizeof(*s->tab_slice_address));
172     s->qp_y_tab           = av_malloc(pic_size_in_ctb *
173                                       sizeof(*s->qp_y_tab));
174     if (!s->qp_y_tab || !s->filter_slice_edges || !s->tab_slice_address)
175         goto fail;
176
177     s->horizontal_bs = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
178     s->vertical_bs   = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
179     if (!s->horizontal_bs || !s->vertical_bs)
180         goto fail;
181
182     s->tab_mvf_pool = av_buffer_pool_init(min_pu_size * sizeof(MvField),
183                                           av_buffer_alloc);
184     s->rpl_tab_pool = av_buffer_pool_init(ctb_count * sizeof(RefPicListTab),
185                                           av_buffer_allocz);
186     if (!s->tab_mvf_pool || !s->rpl_tab_pool)
187         goto fail;
188
189     return 0;
190
191 fail:
192     pic_arrays_free(s);
193     return AVERROR(ENOMEM);
194 }
195
196 static void pred_weight_table(HEVCContext *s, GetBitContext *gb)
197 {
198     int i = 0;
199     int j = 0;
200     uint8_t luma_weight_l0_flag[16];
201     uint8_t chroma_weight_l0_flag[16];
202     uint8_t luma_weight_l1_flag[16];
203     uint8_t chroma_weight_l1_flag[16];
204
205     s->sh.luma_log2_weight_denom = av_clip(get_ue_golomb_long(gb), 0, 7);
206     if (s->ps.sps->chroma_format_idc != 0) {
207         int delta = get_se_golomb(gb);
208         s->sh.chroma_log2_weight_denom = av_clip(s->sh.luma_log2_weight_denom + delta, 0, 7);
209     }
210
211     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
212         luma_weight_l0_flag[i] = get_bits1(gb);
213         if (!luma_weight_l0_flag[i]) {
214             s->sh.luma_weight_l0[i] = 1 << s->sh.luma_log2_weight_denom;
215             s->sh.luma_offset_l0[i] = 0;
216         }
217     }
218     if (s->ps.sps->chroma_format_idc != 0) { // FIXME: invert "if" and "for"
219         for (i = 0; i < s->sh.nb_refs[L0]; i++)
220             chroma_weight_l0_flag[i] = get_bits1(gb);
221     } else {
222         for (i = 0; i < s->sh.nb_refs[L0]; i++)
223             chroma_weight_l0_flag[i] = 0;
224     }
225     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
226         if (luma_weight_l0_flag[i]) {
227             int delta_luma_weight_l0 = get_se_golomb(gb);
228             s->sh.luma_weight_l0[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l0;
229             s->sh.luma_offset_l0[i] = get_se_golomb(gb);
230         }
231         if (chroma_weight_l0_flag[i]) {
232             for (j = 0; j < 2; j++) {
233                 int delta_chroma_weight_l0 = get_se_golomb(gb);
234                 int delta_chroma_offset_l0 = get_se_golomb(gb);
235                 s->sh.chroma_weight_l0[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l0;
236                 s->sh.chroma_offset_l0[i][j] = av_clip((delta_chroma_offset_l0 - ((128 * s->sh.chroma_weight_l0[i][j])
237                                                                                     >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
238             }
239         } else {
240             s->sh.chroma_weight_l0[i][0] = 1 << s->sh.chroma_log2_weight_denom;
241             s->sh.chroma_offset_l0[i][0] = 0;
242             s->sh.chroma_weight_l0[i][1] = 1 << s->sh.chroma_log2_weight_denom;
243             s->sh.chroma_offset_l0[i][1] = 0;
244         }
245     }
246     if (s->sh.slice_type == HEVC_SLICE_B) {
247         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
248             luma_weight_l1_flag[i] = get_bits1(gb);
249             if (!luma_weight_l1_flag[i]) {
250                 s->sh.luma_weight_l1[i] = 1 << s->sh.luma_log2_weight_denom;
251                 s->sh.luma_offset_l1[i] = 0;
252             }
253         }
254         if (s->ps.sps->chroma_format_idc != 0) {
255             for (i = 0; i < s->sh.nb_refs[L1]; i++)
256                 chroma_weight_l1_flag[i] = get_bits1(gb);
257         } else {
258             for (i = 0; i < s->sh.nb_refs[L1]; i++)
259                 chroma_weight_l1_flag[i] = 0;
260         }
261         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
262             if (luma_weight_l1_flag[i]) {
263                 int delta_luma_weight_l1 = get_se_golomb(gb);
264                 s->sh.luma_weight_l1[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l1;
265                 s->sh.luma_offset_l1[i] = get_se_golomb(gb);
266             }
267             if (chroma_weight_l1_flag[i]) {
268                 for (j = 0; j < 2; j++) {
269                     int delta_chroma_weight_l1 = get_se_golomb(gb);
270                     int delta_chroma_offset_l1 = get_se_golomb(gb);
271                     s->sh.chroma_weight_l1[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l1;
272                     s->sh.chroma_offset_l1[i][j] = av_clip((delta_chroma_offset_l1 - ((128 * s->sh.chroma_weight_l1[i][j])
273                                                                                         >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
274                 }
275             } else {
276                 s->sh.chroma_weight_l1[i][0] = 1 << s->sh.chroma_log2_weight_denom;
277                 s->sh.chroma_offset_l1[i][0] = 0;
278                 s->sh.chroma_weight_l1[i][1] = 1 << s->sh.chroma_log2_weight_denom;
279                 s->sh.chroma_offset_l1[i][1] = 0;
280             }
281         }
282     }
283 }
284
285 static int decode_lt_rps(HEVCContext *s, LongTermRPS *rps, GetBitContext *gb)
286 {
287     const HEVCSPS *sps = s->ps.sps;
288     int max_poc_lsb    = 1 << sps->log2_max_poc_lsb;
289     int prev_delta_msb = 0;
290     unsigned int nb_sps = 0, nb_sh;
291     int i;
292
293     rps->nb_refs = 0;
294     if (!sps->long_term_ref_pics_present_flag)
295         return 0;
296
297     if (sps->num_long_term_ref_pics_sps > 0)
298         nb_sps = get_ue_golomb_long(gb);
299     nb_sh = get_ue_golomb_long(gb);
300
301     if (nb_sh + nb_sps > FF_ARRAY_ELEMS(rps->poc))
302         return AVERROR_INVALIDDATA;
303
304     rps->nb_refs = nb_sh + nb_sps;
305
306     for (i = 0; i < rps->nb_refs; i++) {
307         uint8_t delta_poc_msb_present;
308
309         if (i < nb_sps) {
310             uint8_t lt_idx_sps = 0;
311
312             if (sps->num_long_term_ref_pics_sps > 1)
313                 lt_idx_sps = get_bits(gb, av_ceil_log2(sps->num_long_term_ref_pics_sps));
314
315             rps->poc[i]  = sps->lt_ref_pic_poc_lsb_sps[lt_idx_sps];
316             rps->used[i] = sps->used_by_curr_pic_lt_sps_flag[lt_idx_sps];
317         } else {
318             rps->poc[i]  = get_bits(gb, sps->log2_max_poc_lsb);
319             rps->used[i] = get_bits1(gb);
320         }
321
322         delta_poc_msb_present = get_bits1(gb);
323         if (delta_poc_msb_present) {
324             int delta = get_ue_golomb_long(gb);
325
326             if (i && i != nb_sps)
327                 delta += prev_delta_msb;
328
329             rps->poc[i] += s->poc - delta * max_poc_lsb - s->sh.pic_order_cnt_lsb;
330             prev_delta_msb = delta;
331         }
332     }
333
334     return 0;
335 }
336
337 static void export_stream_params(AVCodecContext *avctx, const HEVCParamSets *ps,
338                                  const HEVCSPS *sps)
339 {
340     const HEVCVPS *vps = (const HEVCVPS*)ps->vps_list[sps->vps_id]->data;
341     const HEVCWindow *ow = &sps->output_window;
342     unsigned int num = 0, den = 0;
343
344     avctx->pix_fmt             = sps->pix_fmt;
345     avctx->coded_width         = sps->width;
346     avctx->coded_height        = sps->height;
347     avctx->width               = sps->width  - ow->left_offset - ow->right_offset;
348     avctx->height              = sps->height - ow->top_offset  - ow->bottom_offset;
349     avctx->has_b_frames        = sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics;
350     avctx->profile             = sps->ptl.general_ptl.profile_idc;
351     avctx->level               = sps->ptl.general_ptl.level_idc;
352
353     ff_set_sar(avctx, sps->vui.sar);
354
355     if (sps->vui.video_signal_type_present_flag)
356         avctx->color_range = sps->vui.video_full_range_flag ? AVCOL_RANGE_JPEG
357                                                             : AVCOL_RANGE_MPEG;
358     else
359         avctx->color_range = AVCOL_RANGE_MPEG;
360
361     if (sps->vui.colour_description_present_flag) {
362         avctx->color_primaries = sps->vui.colour_primaries;
363         avctx->color_trc       = sps->vui.transfer_characteristic;
364         avctx->colorspace      = sps->vui.matrix_coeffs;
365     } else {
366         avctx->color_primaries = AVCOL_PRI_UNSPECIFIED;
367         avctx->color_trc       = AVCOL_TRC_UNSPECIFIED;
368         avctx->colorspace      = AVCOL_SPC_UNSPECIFIED;
369     }
370
371     if (vps->vps_timing_info_present_flag) {
372         num = vps->vps_num_units_in_tick;
373         den = vps->vps_time_scale;
374     } else if (sps->vui.vui_timing_info_present_flag) {
375         num = sps->vui.vui_num_units_in_tick;
376         den = sps->vui.vui_time_scale;
377     }
378
379     if (num != 0 && den != 0)
380         av_reduce(&avctx->framerate.den, &avctx->framerate.num,
381                   num, den, 1 << 30);
382 }
383
384 static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
385 {
386     #define HWACCEL_MAX (CONFIG_HEVC_DXVA2_HWACCEL + CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \
387                          CONFIG_HEVC_VAAPI_HWACCEL + CONFIG_HEVC_VDPAU_HWACCEL)
388     enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmt = pix_fmts;
389
390     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P ||
391         sps->pix_fmt == AV_PIX_FMT_YUV420P10) {
392 #if CONFIG_HEVC_D3D11VA_HWACCEL
393         *fmt++ = AV_PIX_FMT_D3D11VA_VLD;
394         *fmt++ = AV_PIX_FMT_D3D11;
395 #endif
396 #if CONFIG_HEVC_DXVA2_HWACCEL
397         *fmt++ = AV_PIX_FMT_DXVA2_VLD;
398 #endif
399 #if CONFIG_HEVC_VAAPI_HWACCEL
400         *fmt++ = AV_PIX_FMT_VAAPI;
401 #endif
402     }
403     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P) {
404 #if CONFIG_HEVC_VDPAU_HWACCEL
405         *fmt++ = AV_PIX_FMT_VDPAU;
406 #endif
407     }
408
409     *fmt++ = sps->pix_fmt;
410     *fmt = AV_PIX_FMT_NONE;
411
412     return ff_get_format(s->avctx, pix_fmts);
413 }
414
415 static int set_sps(HEVCContext *s, const HEVCSPS *sps,
416                    enum AVPixelFormat pix_fmt)
417 {
418     int ret;
419
420     pic_arrays_free(s);
421     s->ps.sps = NULL;
422     s->ps.vps = NULL;
423
424     if (!sps)
425         return 0;
426
427     ret = pic_arrays_init(s, sps);
428     if (ret < 0)
429         goto fail;
430
431     export_stream_params(s->avctx, &s->ps, sps);
432
433     s->avctx->pix_fmt = pix_fmt;
434
435     ff_hevc_pred_init(&s->hpc,     sps->bit_depth);
436     ff_hevc_dsp_init (&s->hevcdsp, sps->bit_depth);
437     ff_videodsp_init (&s->vdsp,    sps->bit_depth);
438
439     if (sps->sao_enabled && !s->avctx->hwaccel) {
440         av_frame_unref(s->tmp_frame);
441         ret = ff_get_buffer(s->avctx, s->tmp_frame, AV_GET_BUFFER_FLAG_REF);
442         if (ret < 0)
443             goto fail;
444         s->frame = s->tmp_frame;
445     }
446
447     s->ps.sps = sps;
448     s->ps.vps = (HEVCVPS*) s->ps.vps_list[s->ps.sps->vps_id]->data;
449
450     return 0;
451
452 fail:
453     pic_arrays_free(s);
454     s->ps.sps = NULL;
455     return ret;
456 }
457
458 static int hls_slice_header(HEVCContext *s)
459 {
460     GetBitContext *gb = &s->HEVClc.gb;
461     SliceHeader *sh   = &s->sh;
462     int i, ret;
463
464     // Coded parameters
465     sh->first_slice_in_pic_flag = get_bits1(gb);
466     if ((IS_IDR(s) || IS_BLA(s)) && sh->first_slice_in_pic_flag) {
467         s->seq_decode = (s->seq_decode + 1) & 0xff;
468         s->max_ra     = INT_MAX;
469         if (IS_IDR(s))
470             ff_hevc_clear_refs(s);
471     }
472     if (IS_IRAP(s))
473         sh->no_output_of_prior_pics_flag = get_bits1(gb);
474
475     sh->pps_id = get_ue_golomb_long(gb);
476     if (sh->pps_id >= HEVC_MAX_PPS_COUNT || !s->ps.pps_list[sh->pps_id]) {
477         av_log(s->avctx, AV_LOG_ERROR, "PPS id out of range: %d\n", sh->pps_id);
478         return AVERROR_INVALIDDATA;
479     }
480     if (!sh->first_slice_in_pic_flag &&
481         s->ps.pps != (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data) {
482         av_log(s->avctx, AV_LOG_ERROR, "PPS changed between slices.\n");
483         return AVERROR_INVALIDDATA;
484     }
485     s->ps.pps = (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data;
486
487     if (s->ps.sps != (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data) {
488         const HEVCSPS *sps = (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data;
489         enum AVPixelFormat pix_fmt;
490
491         ff_hevc_clear_refs(s);
492
493         pix_fmt = get_format(s, sps);
494         if (pix_fmt < 0)
495             return pix_fmt;
496
497         ret = set_sps(s, sps, pix_fmt);
498         if (ret < 0)
499             return ret;
500
501         s->seq_decode = (s->seq_decode + 1) & 0xff;
502         s->max_ra     = INT_MAX;
503     }
504
505     sh->dependent_slice_segment_flag = 0;
506     if (!sh->first_slice_in_pic_flag) {
507         int slice_address_length;
508
509         if (s->ps.pps->dependent_slice_segments_enabled_flag)
510             sh->dependent_slice_segment_flag = get_bits1(gb);
511
512         slice_address_length = av_ceil_log2(s->ps.sps->ctb_width *
513                                             s->ps.sps->ctb_height);
514         sh->slice_segment_addr = slice_address_length ? get_bits(gb, slice_address_length) : 0;
515         if (sh->slice_segment_addr >= s->ps.sps->ctb_width * s->ps.sps->ctb_height) {
516             av_log(s->avctx, AV_LOG_ERROR,
517                    "Invalid slice segment address: %u.\n",
518                    sh->slice_segment_addr);
519             return AVERROR_INVALIDDATA;
520         }
521
522         if (!sh->dependent_slice_segment_flag) {
523             sh->slice_addr = sh->slice_segment_addr;
524             s->slice_idx++;
525         }
526     } else {
527         sh->slice_segment_addr = sh->slice_addr = 0;
528         s->slice_idx           = 0;
529         s->slice_initialized   = 0;
530     }
531
532     if (!sh->dependent_slice_segment_flag) {
533         s->slice_initialized = 0;
534
535         for (i = 0; i < s->ps.pps->num_extra_slice_header_bits; i++)
536             skip_bits(gb, 1);  // slice_reserved_undetermined_flag[]
537
538         sh->slice_type = get_ue_golomb_long(gb);
539         if (!(sh->slice_type == HEVC_SLICE_I ||
540               sh->slice_type == HEVC_SLICE_P ||
541               sh->slice_type == HEVC_SLICE_B)) {
542             av_log(s->avctx, AV_LOG_ERROR, "Unknown slice type: %d.\n",
543                    sh->slice_type);
544             return AVERROR_INVALIDDATA;
545         }
546         if (IS_IRAP(s) && sh->slice_type != HEVC_SLICE_I) {
547             av_log(s->avctx, AV_LOG_ERROR, "Inter slices in an IRAP frame.\n");
548             return AVERROR_INVALIDDATA;
549         }
550
551         // when flag is not present, picture is inferred to be output
552         sh->pic_output_flag = 1;
553         if (s->ps.pps->output_flag_present_flag)
554             sh->pic_output_flag = get_bits1(gb);
555
556         if (s->ps.sps->separate_colour_plane_flag)
557             sh->colour_plane_id = get_bits(gb, 2);
558
559         if (!IS_IDR(s)) {
560             int poc, pos;
561
562             sh->pic_order_cnt_lsb = get_bits(gb, s->ps.sps->log2_max_poc_lsb);
563             poc = ff_hevc_compute_poc(s, sh->pic_order_cnt_lsb);
564             if (!sh->first_slice_in_pic_flag && poc != s->poc) {
565                 av_log(s->avctx, AV_LOG_WARNING,
566                        "Ignoring POC change between slices: %d -> %d\n", s->poc, poc);
567                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
568                     return AVERROR_INVALIDDATA;
569                 poc = s->poc;
570             }
571             s->poc = poc;
572
573             sh->short_term_ref_pic_set_sps_flag = get_bits1(gb);
574             pos = get_bits_left(gb);
575             if (!sh->short_term_ref_pic_set_sps_flag) {
576                 ret = ff_hevc_decode_short_term_rps(gb, s->avctx, &sh->slice_rps, s->ps.sps, 1);
577                 if (ret < 0)
578                     return ret;
579
580                 sh->short_term_rps = &sh->slice_rps;
581             } else {
582                 int numbits, rps_idx;
583
584                 if (!s->ps.sps->nb_st_rps) {
585                     av_log(s->avctx, AV_LOG_ERROR, "No ref lists in the SPS.\n");
586                     return AVERROR_INVALIDDATA;
587                 }
588
589                 numbits = av_ceil_log2(s->ps.sps->nb_st_rps);
590                 rps_idx = numbits > 0 ? get_bits(gb, numbits) : 0;
591                 sh->short_term_rps = &s->ps.sps->st_rps[rps_idx];
592             }
593             sh->short_term_ref_pic_set_size = pos - get_bits_left(gb);
594
595             pos = get_bits_left(gb);
596             ret = decode_lt_rps(s, &sh->long_term_rps, gb);
597             if (ret < 0) {
598                 av_log(s->avctx, AV_LOG_WARNING, "Invalid long term RPS.\n");
599                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
600                     return AVERROR_INVALIDDATA;
601             }
602             sh->long_term_ref_pic_set_size = pos - get_bits_left(gb);
603
604             if (s->ps.sps->sps_temporal_mvp_enabled_flag)
605                 sh->slice_temporal_mvp_enabled_flag = get_bits1(gb);
606             else
607                 sh->slice_temporal_mvp_enabled_flag = 0;
608         } else {
609             s->sh.short_term_rps = NULL;
610             s->poc               = 0;
611         }
612
613         /* 8.3.1 */
614         if (s->temporal_id == 0 &&
615             s->nal_unit_type != HEVC_NAL_TRAIL_N &&
616             s->nal_unit_type != HEVC_NAL_TSA_N   &&
617             s->nal_unit_type != HEVC_NAL_STSA_N  &&
618             s->nal_unit_type != HEVC_NAL_RADL_N  &&
619             s->nal_unit_type != HEVC_NAL_RADL_R  &&
620             s->nal_unit_type != HEVC_NAL_RASL_N  &&
621             s->nal_unit_type != HEVC_NAL_RASL_R)
622             s->pocTid0 = s->poc;
623
624         if (s->ps.sps->sao_enabled) {
625             sh->slice_sample_adaptive_offset_flag[0] = get_bits1(gb);
626             sh->slice_sample_adaptive_offset_flag[1] =
627             sh->slice_sample_adaptive_offset_flag[2] = get_bits1(gb);
628         } else {
629             sh->slice_sample_adaptive_offset_flag[0] = 0;
630             sh->slice_sample_adaptive_offset_flag[1] = 0;
631             sh->slice_sample_adaptive_offset_flag[2] = 0;
632         }
633
634         sh->nb_refs[L0] = sh->nb_refs[L1] = 0;
635         if (sh->slice_type == HEVC_SLICE_P || sh->slice_type == HEVC_SLICE_B) {
636             int nb_refs;
637
638             sh->nb_refs[L0] = s->ps.pps->num_ref_idx_l0_default_active;
639             if (sh->slice_type == HEVC_SLICE_B)
640                 sh->nb_refs[L1] = s->ps.pps->num_ref_idx_l1_default_active;
641
642             if (get_bits1(gb)) { // num_ref_idx_active_override_flag
643                 sh->nb_refs[L0] = get_ue_golomb_long(gb) + 1;
644                 if (sh->slice_type == HEVC_SLICE_B)
645                     sh->nb_refs[L1] = get_ue_golomb_long(gb) + 1;
646             }
647             if (sh->nb_refs[L0] > HEVC_MAX_REFS || sh->nb_refs[L1] > HEVC_MAX_REFS) {
648                 av_log(s->avctx, AV_LOG_ERROR, "Too many refs: %d/%d.\n",
649                        sh->nb_refs[L0], sh->nb_refs[L1]);
650                 return AVERROR_INVALIDDATA;
651             }
652
653             sh->rpl_modification_flag[0] = 0;
654             sh->rpl_modification_flag[1] = 0;
655             nb_refs = ff_hevc_frame_nb_refs(s);
656             if (!nb_refs) {
657                 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
658                 return AVERROR_INVALIDDATA;
659             }
660
661             if (s->ps.pps->lists_modification_present_flag && nb_refs > 1) {
662                 sh->rpl_modification_flag[0] = get_bits1(gb);
663                 if (sh->rpl_modification_flag[0]) {
664                     for (i = 0; i < sh->nb_refs[L0]; i++)
665                         sh->list_entry_lx[0][i] = get_bits(gb, av_ceil_log2(nb_refs));
666                 }
667
668                 if (sh->slice_type == HEVC_SLICE_B) {
669                     sh->rpl_modification_flag[1] = get_bits1(gb);
670                     if (sh->rpl_modification_flag[1] == 1)
671                         for (i = 0; i < sh->nb_refs[L1]; i++)
672                             sh->list_entry_lx[1][i] = get_bits(gb, av_ceil_log2(nb_refs));
673                 }
674             }
675
676             if (sh->slice_type == HEVC_SLICE_B)
677                 sh->mvd_l1_zero_flag = get_bits1(gb);
678
679             if (s->ps.pps->cabac_init_present_flag)
680                 sh->cabac_init_flag = get_bits1(gb);
681             else
682                 sh->cabac_init_flag = 0;
683
684             sh->collocated_ref_idx = 0;
685             if (sh->slice_temporal_mvp_enabled_flag) {
686                 sh->collocated_list = L0;
687                 if (sh->slice_type == HEVC_SLICE_B)
688                     sh->collocated_list = !get_bits1(gb);
689
690                 if (sh->nb_refs[sh->collocated_list] > 1) {
691                     sh->collocated_ref_idx = get_ue_golomb_long(gb);
692                     if (sh->collocated_ref_idx >= sh->nb_refs[sh->collocated_list]) {
693                         av_log(s->avctx, AV_LOG_ERROR,
694                                "Invalid collocated_ref_idx: %d.\n",
695                                sh->collocated_ref_idx);
696                         return AVERROR_INVALIDDATA;
697                     }
698                 }
699             }
700
701             if ((s->ps.pps->weighted_pred_flag   && sh->slice_type == HEVC_SLICE_P) ||
702                 (s->ps.pps->weighted_bipred_flag && sh->slice_type == HEVC_SLICE_B)) {
703                 pred_weight_table(s, gb);
704             }
705
706             sh->max_num_merge_cand = 5 - get_ue_golomb_long(gb);
707             if (sh->max_num_merge_cand < 1 || sh->max_num_merge_cand > 5) {
708                 av_log(s->avctx, AV_LOG_ERROR,
709                        "Invalid number of merging MVP candidates: %d.\n",
710                        sh->max_num_merge_cand);
711                 return AVERROR_INVALIDDATA;
712             }
713         }
714
715         sh->slice_qp_delta = get_se_golomb(gb);
716
717         if (s->ps.pps->pic_slice_level_chroma_qp_offsets_present_flag) {
718             sh->slice_cb_qp_offset = get_se_golomb(gb);
719             sh->slice_cr_qp_offset = get_se_golomb(gb);
720         } else {
721             sh->slice_cb_qp_offset = 0;
722             sh->slice_cr_qp_offset = 0;
723         }
724
725         if (s->ps.pps->deblocking_filter_control_present_flag) {
726             int deblocking_filter_override_flag = 0;
727
728             if (s->ps.pps->deblocking_filter_override_enabled_flag)
729                 deblocking_filter_override_flag = get_bits1(gb);
730
731             if (deblocking_filter_override_flag) {
732                 sh->disable_deblocking_filter_flag = get_bits1(gb);
733                 if (!sh->disable_deblocking_filter_flag) {
734                     sh->beta_offset = get_se_golomb(gb) * 2;
735                     sh->tc_offset   = get_se_golomb(gb) * 2;
736                 }
737             } else {
738                 sh->disable_deblocking_filter_flag = s->ps.pps->disable_dbf;
739                 sh->beta_offset                    = s->ps.pps->beta_offset;
740                 sh->tc_offset                      = s->ps.pps->tc_offset;
741             }
742         } else {
743             sh->disable_deblocking_filter_flag = 0;
744             sh->beta_offset                    = 0;
745             sh->tc_offset                      = 0;
746         }
747
748         if (s->ps.pps->seq_loop_filter_across_slices_enabled_flag &&
749             (sh->slice_sample_adaptive_offset_flag[0] ||
750              sh->slice_sample_adaptive_offset_flag[1] ||
751              !sh->disable_deblocking_filter_flag)) {
752             sh->slice_loop_filter_across_slices_enabled_flag = get_bits1(gb);
753         } else {
754             sh->slice_loop_filter_across_slices_enabled_flag = s->ps.pps->seq_loop_filter_across_slices_enabled_flag;
755         }
756     } else if (!s->slice_initialized) {
757         av_log(s->avctx, AV_LOG_ERROR, "Independent slice segment missing.\n");
758         return AVERROR_INVALIDDATA;
759     }
760
761     sh->num_entry_point_offsets = 0;
762     if (s->ps.pps->tiles_enabled_flag || s->ps.pps->entropy_coding_sync_enabled_flag) {
763         sh->num_entry_point_offsets = get_ue_golomb_long(gb);
764         if (sh->num_entry_point_offsets > 0) {
765             int offset_len = get_ue_golomb_long(gb) + 1;
766
767             for (i = 0; i < sh->num_entry_point_offsets; i++)
768                 skip_bits(gb, offset_len);
769         }
770     }
771
772     if (s->ps.pps->slice_header_extension_present_flag) {
773         unsigned int length = get_ue_golomb_long(gb);
774         for (i = 0; i < length; i++)
775             skip_bits(gb, 8);  // slice_header_extension_data_byte
776     }
777
778     // Inferred parameters
779     sh->slice_qp = 26 + s->ps.pps->pic_init_qp_minus26 + sh->slice_qp_delta;
780     if (sh->slice_qp > 51 ||
781         sh->slice_qp < -s->ps.sps->qp_bd_offset) {
782         av_log(s->avctx, AV_LOG_ERROR,
783                "The slice_qp %d is outside the valid range "
784                "[%d, 51].\n",
785                sh->slice_qp,
786                -s->ps.sps->qp_bd_offset);
787         return AVERROR_INVALIDDATA;
788     }
789
790     sh->slice_ctb_addr_rs = sh->slice_segment_addr;
791
792     if (!s->sh.slice_ctb_addr_rs && s->sh.dependent_slice_segment_flag) {
793         av_log(s->avctx, AV_LOG_ERROR, "Impossible slice segment.\n");
794         return AVERROR_INVALIDDATA;
795     }
796
797     s->HEVClc.first_qp_group = !s->sh.dependent_slice_segment_flag;
798
799     if (!s->ps.pps->cu_qp_delta_enabled_flag)
800         s->HEVClc.qp_y = FFUMOD(s->sh.slice_qp + 52 + 2 * s->ps.sps->qp_bd_offset,
801                                 52 + s->ps.sps->qp_bd_offset) - s->ps.sps->qp_bd_offset;
802
803     s->slice_initialized = 1;
804
805     return 0;
806 }
807
808 #define CTB(tab, x, y) ((tab)[(y) * s->ps.sps->ctb_width + (x)])
809
810 #define SET_SAO(elem, value)                            \
811 do {                                                    \
812     if (!sao_merge_up_flag && !sao_merge_left_flag)     \
813         sao->elem = value;                              \
814     else if (sao_merge_left_flag)                       \
815         sao->elem = CTB(s->sao, rx-1, ry).elem;         \
816     else if (sao_merge_up_flag)                         \
817         sao->elem = CTB(s->sao, rx, ry-1).elem;         \
818     else                                                \
819         sao->elem = 0;                                  \
820 } while (0)
821
822 static void hls_sao_param(HEVCContext *s, int rx, int ry)
823 {
824     HEVCLocalContext *lc    = &s->HEVClc;
825     int sao_merge_left_flag = 0;
826     int sao_merge_up_flag   = 0;
827     int shift               = s->ps.sps->bit_depth - FFMIN(s->ps.sps->bit_depth, 10);
828     SAOParams *sao          = &CTB(s->sao, rx, ry);
829     int c_idx, i;
830
831     if (s->sh.slice_sample_adaptive_offset_flag[0] ||
832         s->sh.slice_sample_adaptive_offset_flag[1]) {
833         if (rx > 0) {
834             if (lc->ctb_left_flag)
835                 sao_merge_left_flag = ff_hevc_sao_merge_flag_decode(s);
836         }
837         if (ry > 0 && !sao_merge_left_flag) {
838             if (lc->ctb_up_flag)
839                 sao_merge_up_flag = ff_hevc_sao_merge_flag_decode(s);
840         }
841     }
842
843     for (c_idx = 0; c_idx < 3; c_idx++) {
844         if (!s->sh.slice_sample_adaptive_offset_flag[c_idx]) {
845             sao->type_idx[c_idx] = SAO_NOT_APPLIED;
846             continue;
847         }
848
849         if (c_idx == 2) {
850             sao->type_idx[2] = sao->type_idx[1];
851             sao->eo_class[2] = sao->eo_class[1];
852         } else {
853             SET_SAO(type_idx[c_idx], ff_hevc_sao_type_idx_decode(s));
854         }
855
856         if (sao->type_idx[c_idx] == SAO_NOT_APPLIED)
857             continue;
858
859         for (i = 0; i < 4; i++)
860             SET_SAO(offset_abs[c_idx][i], ff_hevc_sao_offset_abs_decode(s));
861
862         if (sao->type_idx[c_idx] == SAO_BAND) {
863             for (i = 0; i < 4; i++) {
864                 if (sao->offset_abs[c_idx][i]) {
865                     SET_SAO(offset_sign[c_idx][i],
866                             ff_hevc_sao_offset_sign_decode(s));
867                 } else {
868                     sao->offset_sign[c_idx][i] = 0;
869                 }
870             }
871             SET_SAO(band_position[c_idx], ff_hevc_sao_band_position_decode(s));
872         } else if (c_idx != 2) {
873             SET_SAO(eo_class[c_idx], ff_hevc_sao_eo_class_decode(s));
874         }
875
876         // Inferred parameters
877         sao->offset_val[c_idx][0] = 0;
878         for (i = 0; i < 4; i++) {
879             sao->offset_val[c_idx][i + 1] = sao->offset_abs[c_idx][i] << shift;
880             if (sao->type_idx[c_idx] == SAO_EDGE) {
881                 if (i > 1)
882                     sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
883             } else if (sao->offset_sign[c_idx][i]) {
884                 sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
885             }
886         }
887     }
888 }
889
890 #undef SET_SAO
891 #undef CTB
892
893 static void hls_residual_coding(HEVCContext *s, int x0, int y0,
894                                 int log2_trafo_size, enum ScanType scan_idx,
895                                 int c_idx)
896 {
897 #define GET_COORD(offset, n)                                    \
898     do {                                                        \
899         x_c = (scan_x_cg[offset >> 4] << 2) + scan_x_off[n];    \
900         y_c = (scan_y_cg[offset >> 4] << 2) + scan_y_off[n];    \
901     } while (0)
902     HEVCLocalContext *lc    = &s->HEVClc;
903     int transform_skip_flag = 0;
904
905     int last_significant_coeff_x, last_significant_coeff_y;
906     int last_scan_pos;
907     int n_end;
908     int num_coeff    = 0;
909     int greater1_ctx = 1;
910
911     int num_last_subset;
912     int x_cg_last_sig, y_cg_last_sig;
913
914     const uint8_t *scan_x_cg, *scan_y_cg, *scan_x_off, *scan_y_off;
915
916     ptrdiff_t stride = s->frame->linesize[c_idx];
917     int hshift       = s->ps.sps->hshift[c_idx];
918     int vshift       = s->ps.sps->vshift[c_idx];
919     uint8_t *dst     = &s->frame->data[c_idx][(y0 >> vshift) * stride +
920                                               ((x0 >> hshift) << s->ps.sps->pixel_shift)];
921     LOCAL_ALIGNED_32(int16_t, coeffs, [MAX_TB_SIZE * MAX_TB_SIZE]);
922     LOCAL_ALIGNED_8(uint8_t, significant_coeff_group_flag, [8], [8]);
923
924     int trafo_size = 1 << log2_trafo_size;
925     int i, qp, shift, add, scale, scale_m;
926     static const uint8_t level_scale[] = { 40, 45, 51, 57, 64, 72 };
927     const uint8_t *scale_matrix;
928     uint8_t dc_scale;
929
930     memset(coeffs, 0, sizeof(int16_t) * MAX_TB_SIZE * MAX_TB_SIZE);
931     memset(significant_coeff_group_flag, 0, sizeof(uint8_t) * 8 * 8);
932     // Derive QP for dequant
933     if (!lc->cu.cu_transquant_bypass_flag) {
934         static const int qp_c[] = {
935             29, 30, 31, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37
936         };
937
938         static const uint8_t rem6[51 + 2 * 6 + 1] = {
939             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
940             3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
941             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
942         };
943
944         static const uint8_t div6[51 + 2 * 6 + 1] = {
945             0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,  3,  3,  3,
946             3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6,  6,  6,  6,
947             7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10,
948         };
949         int qp_y = lc->qp_y;
950
951         if (c_idx == 0) {
952             qp = qp_y + s->ps.sps->qp_bd_offset;
953         } else {
954             int qp_i, offset;
955
956             if (c_idx == 1)
957                 offset = s->ps.pps->cb_qp_offset + s->sh.slice_cb_qp_offset;
958             else
959                 offset = s->ps.pps->cr_qp_offset + s->sh.slice_cr_qp_offset;
960
961             qp_i = av_clip(qp_y + offset, -s->ps.sps->qp_bd_offset, 57);
962             if (qp_i < 30)
963                 qp = qp_i;
964             else if (qp_i > 43)
965                 qp = qp_i - 6;
966             else
967                 qp = qp_c[qp_i - 30];
968
969             qp += s->ps.sps->qp_bd_offset;
970         }
971
972         shift    = s->ps.sps->bit_depth + log2_trafo_size - 5;
973         add      = 1 << (shift - 1);
974         scale    = level_scale[rem6[qp]] << (div6[qp]);
975         scale_m  = 16; // default when no custom scaling lists.
976         dc_scale = 16;
977
978         if (s->ps.sps->scaling_list_enable_flag) {
979             const ScalingList *sl = s->ps.pps->scaling_list_data_present_flag ?
980                                     &s->ps.pps->scaling_list : &s->ps.sps->scaling_list;
981             int matrix_id = lc->cu.pred_mode != MODE_INTRA;
982
983             if (log2_trafo_size != 5)
984                 matrix_id = 3 * matrix_id + c_idx;
985
986             scale_matrix = sl->sl[log2_trafo_size - 2][matrix_id];
987             if (log2_trafo_size >= 4)
988                 dc_scale = sl->sl_dc[log2_trafo_size - 4][matrix_id];
989         }
990     }
991
992     if (s->ps.pps->transform_skip_enabled_flag &&
993         !lc->cu.cu_transquant_bypass_flag   &&
994         log2_trafo_size == 2) {
995         transform_skip_flag = ff_hevc_transform_skip_flag_decode(s, c_idx);
996     }
997
998     last_significant_coeff_x =
999         ff_hevc_last_significant_coeff_x_prefix_decode(s, c_idx, log2_trafo_size);
1000     last_significant_coeff_y =
1001         ff_hevc_last_significant_coeff_y_prefix_decode(s, c_idx, log2_trafo_size);
1002
1003     if (last_significant_coeff_x > 3) {
1004         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_x);
1005         last_significant_coeff_x = (1 << ((last_significant_coeff_x >> 1) - 1)) *
1006                                    (2 + (last_significant_coeff_x & 1)) +
1007                                    suffix;
1008     }
1009
1010     if (last_significant_coeff_y > 3) {
1011         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_y);
1012         last_significant_coeff_y = (1 << ((last_significant_coeff_y >> 1) - 1)) *
1013                                    (2 + (last_significant_coeff_y & 1)) +
1014                                    suffix;
1015     }
1016
1017     if (scan_idx == SCAN_VERT)
1018         FFSWAP(int, last_significant_coeff_x, last_significant_coeff_y);
1019
1020     x_cg_last_sig = last_significant_coeff_x >> 2;
1021     y_cg_last_sig = last_significant_coeff_y >> 2;
1022
1023     switch (scan_idx) {
1024     case SCAN_DIAG: {
1025         int last_x_c = last_significant_coeff_x & 3;
1026         int last_y_c = last_significant_coeff_y & 3;
1027
1028         scan_x_off = ff_hevc_diag_scan4x4_x;
1029         scan_y_off = ff_hevc_diag_scan4x4_y;
1030         num_coeff  = diag_scan4x4_inv[last_y_c][last_x_c];
1031         if (trafo_size == 4) {
1032             scan_x_cg = scan_1x1;
1033             scan_y_cg = scan_1x1;
1034         } else if (trafo_size == 8) {
1035             num_coeff += diag_scan2x2_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1036             scan_x_cg  = diag_scan2x2_x;
1037             scan_y_cg  = diag_scan2x2_y;
1038         } else if (trafo_size == 16) {
1039             num_coeff += diag_scan4x4_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1040             scan_x_cg  = ff_hevc_diag_scan4x4_x;
1041             scan_y_cg  = ff_hevc_diag_scan4x4_y;
1042         } else { // trafo_size == 32
1043             num_coeff += diag_scan8x8_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1044             scan_x_cg  = ff_hevc_diag_scan8x8_x;
1045             scan_y_cg  = ff_hevc_diag_scan8x8_y;
1046         }
1047         break;
1048     }
1049     case SCAN_HORIZ:
1050         scan_x_cg  = horiz_scan2x2_x;
1051         scan_y_cg  = horiz_scan2x2_y;
1052         scan_x_off = horiz_scan4x4_x;
1053         scan_y_off = horiz_scan4x4_y;
1054         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_y][last_significant_coeff_x];
1055         break;
1056     default: //SCAN_VERT
1057         scan_x_cg  = horiz_scan2x2_y;
1058         scan_y_cg  = horiz_scan2x2_x;
1059         scan_x_off = horiz_scan4x4_y;
1060         scan_y_off = horiz_scan4x4_x;
1061         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_x][last_significant_coeff_y];
1062         break;
1063     }
1064     num_coeff++;
1065     num_last_subset = (num_coeff - 1) >> 4;
1066
1067     for (i = num_last_subset; i >= 0; i--) {
1068         int n, m;
1069         int x_cg, y_cg, x_c, y_c;
1070         int implicit_non_zero_coeff = 0;
1071         int64_t trans_coeff_level;
1072         int prev_sig = 0;
1073         int offset   = i << 4;
1074
1075         uint8_t significant_coeff_flag_idx[16];
1076         uint8_t nb_significant_coeff_flag = 0;
1077
1078         x_cg = scan_x_cg[i];
1079         y_cg = scan_y_cg[i];
1080
1081         if (i < num_last_subset && i > 0) {
1082             int ctx_cg = 0;
1083             if (x_cg < (1 << (log2_trafo_size - 2)) - 1)
1084                 ctx_cg += significant_coeff_group_flag[x_cg + 1][y_cg];
1085             if (y_cg < (1 << (log2_trafo_size - 2)) - 1)
1086                 ctx_cg += significant_coeff_group_flag[x_cg][y_cg + 1];
1087
1088             significant_coeff_group_flag[x_cg][y_cg] =
1089                 ff_hevc_significant_coeff_group_flag_decode(s, c_idx, ctx_cg);
1090             implicit_non_zero_coeff = 1;
1091         } else {
1092             significant_coeff_group_flag[x_cg][y_cg] =
1093                 ((x_cg == x_cg_last_sig && y_cg == y_cg_last_sig) ||
1094                  (x_cg == 0 && y_cg == 0));
1095         }
1096
1097         last_scan_pos = num_coeff - offset - 1;
1098
1099         if (i == num_last_subset) {
1100             n_end                         = last_scan_pos - 1;
1101             significant_coeff_flag_idx[0] = last_scan_pos;
1102             nb_significant_coeff_flag     = 1;
1103         } else {
1104             n_end = 15;
1105         }
1106
1107         if (x_cg < ((1 << log2_trafo_size) - 1) >> 2)
1108             prev_sig = significant_coeff_group_flag[x_cg + 1][y_cg];
1109         if (y_cg < ((1 << log2_trafo_size) - 1) >> 2)
1110             prev_sig += significant_coeff_group_flag[x_cg][y_cg + 1] << 1;
1111
1112         for (n = n_end; n >= 0; n--) {
1113             GET_COORD(offset, n);
1114
1115             if (significant_coeff_group_flag[x_cg][y_cg] &&
1116                 (n > 0 || implicit_non_zero_coeff == 0)) {
1117                 if (ff_hevc_significant_coeff_flag_decode(s, c_idx, x_c, y_c,
1118                                                           log2_trafo_size,
1119                                                           scan_idx,
1120                                                           prev_sig) == 1) {
1121                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1122                     nb_significant_coeff_flag++;
1123                     implicit_non_zero_coeff = 0;
1124                 }
1125             } else {
1126                 int last_cg = (x_c == (x_cg << 2) && y_c == (y_cg << 2));
1127                 if (last_cg && implicit_non_zero_coeff && significant_coeff_group_flag[x_cg][y_cg]) {
1128                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1129                     nb_significant_coeff_flag++;
1130                 }
1131             }
1132         }
1133
1134         n_end = nb_significant_coeff_flag;
1135
1136         if (n_end) {
1137             int first_nz_pos_in_cg = 16;
1138             int last_nz_pos_in_cg = -1;
1139             int c_rice_param = 0;
1140             int first_greater1_coeff_idx = -1;
1141             uint8_t coeff_abs_level_greater1_flag[16] = { 0 };
1142             uint16_t coeff_sign_flag;
1143             int sum_abs = 0;
1144             int sign_hidden = 0;
1145
1146             // initialize first elem of coeff_bas_level_greater1_flag
1147             int ctx_set = (i > 0 && c_idx == 0) ? 2 : 0;
1148
1149             if (!(i == num_last_subset) && greater1_ctx == 0)
1150                 ctx_set++;
1151             greater1_ctx      = 1;
1152             last_nz_pos_in_cg = significant_coeff_flag_idx[0];
1153
1154             for (m = 0; m < (n_end > 8 ? 8 : n_end); m++) {
1155                 int n_idx = significant_coeff_flag_idx[m];
1156                 int inc   = (ctx_set << 2) + greater1_ctx;
1157                 coeff_abs_level_greater1_flag[n_idx] =
1158                     ff_hevc_coeff_abs_level_greater1_flag_decode(s, c_idx, inc);
1159                 if (coeff_abs_level_greater1_flag[n_idx]) {
1160                     greater1_ctx = 0;
1161                 } else if (greater1_ctx > 0 && greater1_ctx < 3) {
1162                     greater1_ctx++;
1163                 }
1164
1165                 if (coeff_abs_level_greater1_flag[n_idx] &&
1166                     first_greater1_coeff_idx == -1)
1167                     first_greater1_coeff_idx = n_idx;
1168             }
1169             first_nz_pos_in_cg = significant_coeff_flag_idx[n_end - 1];
1170             sign_hidden        = last_nz_pos_in_cg - first_nz_pos_in_cg >= 4 &&
1171                                  !lc->cu.cu_transquant_bypass_flag;
1172
1173             if (first_greater1_coeff_idx != -1) {
1174                 coeff_abs_level_greater1_flag[first_greater1_coeff_idx] += ff_hevc_coeff_abs_level_greater2_flag_decode(s, c_idx, ctx_set);
1175             }
1176             if (!s->ps.pps->sign_data_hiding_flag || !sign_hidden) {
1177                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag) << (16 - nb_significant_coeff_flag);
1178             } else {
1179                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag - 1) << (16 - (nb_significant_coeff_flag - 1));
1180             }
1181
1182             for (m = 0; m < n_end; m++) {
1183                 n = significant_coeff_flag_idx[m];
1184                 GET_COORD(offset, n);
1185                 trans_coeff_level = 1 + coeff_abs_level_greater1_flag[n];
1186                 if (trans_coeff_level == ((m < 8) ?
1187                                           ((n == first_greater1_coeff_idx) ? 3 : 2) : 1)) {
1188                     trans_coeff_level += ff_hevc_coeff_abs_level_remaining(s, trans_coeff_level, c_rice_param);
1189                     if ((trans_coeff_level) > (3 * (1 << c_rice_param)))
1190                         c_rice_param = FFMIN(c_rice_param + 1, 4);
1191                 }
1192                 if (s->ps.pps->sign_data_hiding_flag && sign_hidden) {
1193                     sum_abs += trans_coeff_level;
1194                     if (n == first_nz_pos_in_cg && ((sum_abs & 1) == 1))
1195                         trans_coeff_level = -trans_coeff_level;
1196                 }
1197                 if (coeff_sign_flag >> 15)
1198                     trans_coeff_level = -trans_coeff_level;
1199                 coeff_sign_flag <<= 1;
1200                 if (!lc->cu.cu_transquant_bypass_flag) {
1201                     if (s->ps.sps->scaling_list_enable_flag) {
1202                         if (y_c || x_c || log2_trafo_size < 4) {
1203                             int pos;
1204                             switch (log2_trafo_size) {
1205                             case 3:  pos = (y_c        << 3) +  x_c;       break;
1206                             case 4:  pos = ((y_c >> 1) << 3) + (x_c >> 1); break;
1207                             case 5:  pos = ((y_c >> 2) << 3) + (x_c >> 2); break;
1208                             default: pos = (y_c        << 2) +  x_c;
1209                             }
1210                             scale_m = scale_matrix[pos];
1211                         } else {
1212                             scale_m = dc_scale;
1213                         }
1214                     }
1215                     trans_coeff_level = (trans_coeff_level * (int64_t)scale * (int64_t)scale_m + add) >> shift;
1216                     if(trans_coeff_level < 0) {
1217                         if((~trans_coeff_level) & 0xFffffffffff8000)
1218                             trans_coeff_level = -32768;
1219                     } else {
1220                         if (trans_coeff_level & 0xffffffffffff8000)
1221                             trans_coeff_level = 32767;
1222                     }
1223                 }
1224                 coeffs[y_c * trafo_size + x_c] = trans_coeff_level;
1225             }
1226         }
1227     }
1228
1229     if (!lc->cu.cu_transquant_bypass_flag) {
1230         if (transform_skip_flag)
1231             s->hevcdsp.dequant(coeffs);
1232         else if (lc->cu.pred_mode == MODE_INTRA && c_idx == 0 &&
1233                  log2_trafo_size == 2)
1234             s->hevcdsp.transform_4x4_luma(coeffs);
1235         else {
1236             int max_xy = FFMAX(last_significant_coeff_x, last_significant_coeff_y);
1237             if (max_xy == 0)
1238                 s->hevcdsp.idct_dc[log2_trafo_size - 2](coeffs);
1239             else {
1240                 int col_limit = last_significant_coeff_x + last_significant_coeff_y + 4;
1241                 if (max_xy < 4)
1242                     col_limit = FFMIN(4, col_limit);
1243                 else if (max_xy < 8)
1244                     col_limit = FFMIN(8, col_limit);
1245                 else if (max_xy < 12)
1246                     col_limit = FFMIN(24, col_limit);
1247                 s->hevcdsp.idct[log2_trafo_size - 2](coeffs, col_limit);
1248             }
1249         }
1250     }
1251     s->hevcdsp.add_residual[log2_trafo_size - 2](dst, coeffs, stride);
1252 }
1253
1254 static int hls_transform_unit(HEVCContext *s, int x0, int y0,
1255                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1256                               int log2_cb_size, int log2_trafo_size,
1257                               int blk_idx, int cbf_luma, int cbf_cb, int cbf_cr)
1258 {
1259     HEVCLocalContext *lc = &s->HEVClc;
1260
1261     if (lc->cu.pred_mode == MODE_INTRA) {
1262         int trafo_size = 1 << log2_trafo_size;
1263         ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1264
1265         s->hpc.intra_pred[log2_trafo_size - 2](s, x0, y0, 0);
1266         if (log2_trafo_size > 2) {
1267             trafo_size = trafo_size << (s->ps.sps->hshift[1] - 1);
1268             ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1269             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 1);
1270             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 2);
1271         } else if (blk_idx == 3) {
1272             trafo_size = trafo_size << s->ps.sps->hshift[1];
1273             ff_hevc_set_neighbour_available(s, xBase, yBase,
1274                                             trafo_size, trafo_size);
1275             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 1);
1276             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 2);
1277         }
1278     }
1279
1280     if (cbf_luma || cbf_cb || cbf_cr) {
1281         int scan_idx   = SCAN_DIAG;
1282         int scan_idx_c = SCAN_DIAG;
1283
1284         if (s->ps.pps->cu_qp_delta_enabled_flag && !lc->tu.is_cu_qp_delta_coded) {
1285             lc->tu.cu_qp_delta = ff_hevc_cu_qp_delta_abs(s);
1286             if (lc->tu.cu_qp_delta != 0)
1287                 if (ff_hevc_cu_qp_delta_sign_flag(s) == 1)
1288                     lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta;
1289             lc->tu.is_cu_qp_delta_coded = 1;
1290
1291             if (lc->tu.cu_qp_delta < -(26 + s->ps.sps->qp_bd_offset / 2) ||
1292                 lc->tu.cu_qp_delta >  (25 + s->ps.sps->qp_bd_offset / 2)) {
1293                 av_log(s->avctx, AV_LOG_ERROR,
1294                        "The cu_qp_delta %d is outside the valid range "
1295                        "[%d, %d].\n",
1296                        lc->tu.cu_qp_delta,
1297                        -(26 + s->ps.sps->qp_bd_offset / 2),
1298                         (25 + s->ps.sps->qp_bd_offset / 2));
1299                 return AVERROR_INVALIDDATA;
1300             }
1301
1302             ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size);
1303         }
1304
1305         if (lc->cu.pred_mode == MODE_INTRA && log2_trafo_size < 4) {
1306             if (lc->tu.cur_intra_pred_mode >= 6 &&
1307                 lc->tu.cur_intra_pred_mode <= 14) {
1308                 scan_idx = SCAN_VERT;
1309             } else if (lc->tu.cur_intra_pred_mode >= 22 &&
1310                        lc->tu.cur_intra_pred_mode <= 30) {
1311                 scan_idx = SCAN_HORIZ;
1312             }
1313
1314             if (lc->pu.intra_pred_mode_c >=  6 &&
1315                 lc->pu.intra_pred_mode_c <= 14) {
1316                 scan_idx_c = SCAN_VERT;
1317             } else if (lc->pu.intra_pred_mode_c >= 22 &&
1318                        lc->pu.intra_pred_mode_c <= 30) {
1319                 scan_idx_c = SCAN_HORIZ;
1320             }
1321         }
1322
1323         if (cbf_luma)
1324             hls_residual_coding(s, x0, y0, log2_trafo_size, scan_idx, 0);
1325         if (log2_trafo_size > 2) {
1326             if (cbf_cb)
1327                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 1);
1328             if (cbf_cr)
1329                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 2);
1330         } else if (blk_idx == 3) {
1331             if (cbf_cb)
1332                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 1);
1333             if (cbf_cr)
1334                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2);
1335         }
1336     }
1337     return 0;
1338 }
1339
1340 static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size)
1341 {
1342     int cb_size          = 1 << log2_cb_size;
1343     int log2_min_pu_size = s->ps.sps->log2_min_pu_size;
1344
1345     int min_pu_width     = s->ps.sps->min_pu_width;
1346     int x_end = FFMIN(x0 + cb_size, s->ps.sps->width);
1347     int y_end = FFMIN(y0 + cb_size, s->ps.sps->height);
1348     int i, j;
1349
1350     for (j = (y0 >> log2_min_pu_size); j < (y_end >> log2_min_pu_size); j++)
1351         for (i = (x0 >> log2_min_pu_size); i < (x_end >> log2_min_pu_size); i++)
1352             s->is_pcm[i + j * min_pu_width] = 2;
1353 }
1354
1355 static int hls_transform_tree(HEVCContext *s, int x0, int y0,
1356                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1357                               int log2_cb_size, int log2_trafo_size,
1358                               int trafo_depth, int blk_idx,
1359                               int cbf_cb, int cbf_cr)
1360 {
1361     HEVCLocalContext *lc = &s->HEVClc;
1362     uint8_t split_transform_flag;
1363     int ret;
1364
1365     if (lc->cu.intra_split_flag) {
1366         if (trafo_depth == 1)
1367             lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[blk_idx];
1368     } else {
1369         lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[0];
1370     }
1371
1372     if (log2_trafo_size <= s->ps.sps->log2_max_trafo_size &&
1373         log2_trafo_size >  s->ps.sps->log2_min_tb_size    &&
1374         trafo_depth     < lc->cu.max_trafo_depth       &&
1375         !(lc->cu.intra_split_flag && trafo_depth == 0)) {
1376         split_transform_flag = ff_hevc_split_transform_flag_decode(s, log2_trafo_size);
1377     } else {
1378         int inter_split = s->ps.sps->max_transform_hierarchy_depth_inter == 0 &&
1379                           lc->cu.pred_mode == MODE_INTER &&
1380                           lc->cu.part_mode != PART_2Nx2N &&
1381                           trafo_depth == 0;
1382
1383         split_transform_flag = log2_trafo_size > s->ps.sps->log2_max_trafo_size ||
1384                                (lc->cu.intra_split_flag && trafo_depth == 0) ||
1385                                inter_split;
1386     }
1387
1388     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cb))
1389         cbf_cb = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1390     else if (log2_trafo_size > 2 || trafo_depth == 0)
1391         cbf_cb = 0;
1392     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cr))
1393         cbf_cr = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1394     else if (log2_trafo_size > 2 || trafo_depth == 0)
1395         cbf_cr = 0;
1396
1397     if (split_transform_flag) {
1398         const int trafo_size_split = 1 << (log2_trafo_size - 1);
1399         const int x1 = x0 + trafo_size_split;
1400         const int y1 = y0 + trafo_size_split;
1401
1402 #define SUBDIVIDE(x, y, idx)                                                    \
1403 do {                                                                            \
1404     ret = hls_transform_tree(s, x, y, x0, y0, cb_xBase, cb_yBase, log2_cb_size, \
1405                              log2_trafo_size - 1, trafo_depth + 1, idx,         \
1406                              cbf_cb, cbf_cr);                                   \
1407     if (ret < 0)                                                                \
1408         return ret;                                                             \
1409 } while (0)
1410
1411         SUBDIVIDE(x0, y0, 0);
1412         SUBDIVIDE(x1, y0, 1);
1413         SUBDIVIDE(x0, y1, 2);
1414         SUBDIVIDE(x1, y1, 3);
1415
1416 #undef SUBDIVIDE
1417     } else {
1418         int min_tu_size      = 1 << s->ps.sps->log2_min_tb_size;
1419         int log2_min_tu_size = s->ps.sps->log2_min_tb_size;
1420         int min_tu_width     = s->ps.sps->min_tb_width;
1421         int cbf_luma         = 1;
1422
1423         if (lc->cu.pred_mode == MODE_INTRA || trafo_depth != 0 ||
1424             cbf_cb || cbf_cr)
1425             cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth);
1426
1427         ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase,
1428                                  log2_cb_size, log2_trafo_size,
1429                                  blk_idx, cbf_luma, cbf_cb, cbf_cr);
1430         if (ret < 0)
1431             return ret;
1432         // TODO: store cbf_luma somewhere else
1433         if (cbf_luma) {
1434             int i, j;
1435             for (i = 0; i < (1 << log2_trafo_size); i += min_tu_size)
1436                 for (j = 0; j < (1 << log2_trafo_size); j += min_tu_size) {
1437                     int x_tu = (x0 + j) >> log2_min_tu_size;
1438                     int y_tu = (y0 + i) >> log2_min_tu_size;
1439                     s->cbf_luma[y_tu * min_tu_width + x_tu] = 1;
1440                 }
1441         }
1442         if (!s->sh.disable_deblocking_filter_flag) {
1443             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_trafo_size);
1444             if (s->ps.pps->transquant_bypass_enable_flag &&
1445                 lc->cu.cu_transquant_bypass_flag)
1446                 set_deblocking_bypass(s, x0, y0, log2_trafo_size);
1447         }
1448     }
1449     return 0;
1450 }
1451
1452 static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size)
1453 {
1454     //TODO: non-4:2:0 support
1455     HEVCLocalContext *lc = &s->HEVClc;
1456     GetBitContext gb;
1457     int cb_size   = 1 << log2_cb_size;
1458     ptrdiff_t stride0 = s->frame->linesize[0];
1459     ptrdiff_t stride1 = s->frame->linesize[1];
1460     ptrdiff_t stride2 = s->frame->linesize[2];
1461     uint8_t *dst0 = &s->frame->data[0][y0 * stride0 + (x0 << s->ps.sps->pixel_shift)];
1462     uint8_t *dst1 = &s->frame->data[1][(y0 >> s->ps.sps->vshift[1]) * stride1 + ((x0 >> s->ps.sps->hshift[1]) << s->ps.sps->pixel_shift)];
1463     uint8_t *dst2 = &s->frame->data[2][(y0 >> s->ps.sps->vshift[2]) * stride2 + ((x0 >> s->ps.sps->hshift[2]) << s->ps.sps->pixel_shift)];
1464
1465     int length         = cb_size * cb_size * s->ps.sps->pcm.bit_depth + ((cb_size * cb_size) >> 1) * s->ps.sps->pcm.bit_depth_chroma;
1466     const uint8_t *pcm = skip_bytes(&lc->cc, (length + 7) >> 3);
1467     int ret;
1468
1469     if (!s->sh.disable_deblocking_filter_flag)
1470         ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
1471
1472     ret = init_get_bits(&gb, pcm, length);
1473     if (ret < 0)
1474         return ret;
1475
1476     s->hevcdsp.put_pcm(dst0, stride0, cb_size,     &gb, s->ps.sps->pcm.bit_depth);
1477     s->hevcdsp.put_pcm(dst1, stride1, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1478     s->hevcdsp.put_pcm(dst2, stride2, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1479     return 0;
1480 }
1481
1482 static void hls_mvd_coding(HEVCContext *s, int x0, int y0, int log2_cb_size)
1483 {
1484     HEVCLocalContext *lc = &s->HEVClc;
1485     int x = ff_hevc_abs_mvd_greater0_flag_decode(s);
1486     int y = ff_hevc_abs_mvd_greater0_flag_decode(s);
1487
1488     if (x)
1489         x += ff_hevc_abs_mvd_greater1_flag_decode(s);
1490     if (y)
1491         y += ff_hevc_abs_mvd_greater1_flag_decode(s);
1492
1493     switch (x) {
1494     case 2: lc->pu.mvd.x = ff_hevc_mvd_decode(s);           break;
1495     case 1: lc->pu.mvd.x = ff_hevc_mvd_sign_flag_decode(s); break;
1496     case 0: lc->pu.mvd.x = 0;                               break;
1497     }
1498
1499     switch (y) {
1500     case 2: lc->pu.mvd.y = ff_hevc_mvd_decode(s);           break;
1501     case 1: lc->pu.mvd.y = ff_hevc_mvd_sign_flag_decode(s); break;
1502     case 0: lc->pu.mvd.y = 0;                               break;
1503     }
1504 }
1505
1506 /**
1507  * 8.5.3.2.2.1 Luma sample interpolation process
1508  *
1509  * @param s HEVC decoding context
1510  * @param dst target buffer for block data at block position
1511  * @param dststride stride of the dst buffer
1512  * @param ref reference picture buffer at origin (0, 0)
1513  * @param mv motion vector (relative to block position) to get pixel data from
1514  * @param x_off horizontal position of block from origin (0, 0)
1515  * @param y_off vertical position of block from origin (0, 0)
1516  * @param block_w width of block
1517  * @param block_h height of block
1518  */
1519 static void luma_mc(HEVCContext *s, int16_t *dst, ptrdiff_t dststride,
1520                     AVFrame *ref, const Mv *mv, int x_off, int y_off,
1521                     int block_w, int block_h, int pred_idx)
1522 {
1523     HEVCLocalContext *lc = &s->HEVClc;
1524     uint8_t *src         = ref->data[0];
1525     ptrdiff_t srcstride  = ref->linesize[0];
1526     int pic_width        = s->ps.sps->width;
1527     int pic_height       = s->ps.sps->height;
1528
1529     int mx         = mv->x & 3;
1530     int my         = mv->y & 3;
1531     int extra_left = ff_hevc_qpel_extra_before[mx];
1532     int extra_top  = ff_hevc_qpel_extra_before[my];
1533
1534     x_off += mv->x >> 2;
1535     y_off += mv->y >> 2;
1536     src   += y_off * srcstride + (x_off * (1 << s->ps.sps->pixel_shift));
1537
1538     if (x_off < extra_left || y_off < extra_top ||
1539         x_off >= pic_width - block_w - ff_hevc_qpel_extra_after[mx] ||
1540         y_off >= pic_height - block_h - ff_hevc_qpel_extra_after[my]) {
1541         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1542         int offset = extra_top * srcstride + (extra_left << s->ps.sps->pixel_shift);
1543         int buf_offset = extra_top *
1544                          edge_emu_stride + (extra_left << s->ps.sps->pixel_shift);
1545
1546         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src - offset,
1547                                  edge_emu_stride, srcstride,
1548                                  block_w + ff_hevc_qpel_extra[mx],
1549                                  block_h + ff_hevc_qpel_extra[my],
1550                                  x_off - extra_left, y_off - extra_top,
1551                                  pic_width, pic_height);
1552         src = lc->edge_emu_buffer + buf_offset;
1553         srcstride = edge_emu_stride;
1554     }
1555     s->hevcdsp.put_hevc_qpel[!!my][!!mx][pred_idx](dst, dststride, src, srcstride,
1556                                                    block_h, mx, my, lc->mc_buffer);
1557 }
1558
1559 /**
1560  * 8.5.3.2.2.2 Chroma sample interpolation process
1561  *
1562  * @param s HEVC decoding context
1563  * @param dst1 target buffer for block data at block position (U plane)
1564  * @param dst2 target buffer for block data at block position (V plane)
1565  * @param dststride stride of the dst1 and dst2 buffers
1566  * @param ref reference picture buffer at origin (0, 0)
1567  * @param mv motion vector (relative to block position) to get pixel data from
1568  * @param x_off horizontal position of block from origin (0, 0)
1569  * @param y_off vertical position of block from origin (0, 0)
1570  * @param block_w width of block
1571  * @param block_h height of block
1572  */
1573 static void chroma_mc(HEVCContext *s, int16_t *dst1, int16_t *dst2,
1574                       ptrdiff_t dststride, AVFrame *ref, const Mv *mv,
1575                       int x_off, int y_off, int block_w, int block_h, int pred_idx)
1576 {
1577     HEVCLocalContext *lc = &s->HEVClc;
1578     uint8_t *src1        = ref->data[1];
1579     uint8_t *src2        = ref->data[2];
1580     ptrdiff_t src1stride = ref->linesize[1];
1581     ptrdiff_t src2stride = ref->linesize[2];
1582     int pic_width        = s->ps.sps->width >> 1;
1583     int pic_height       = s->ps.sps->height >> 1;
1584
1585     int mx = mv->x & 7;
1586     int my = mv->y & 7;
1587
1588     x_off += mv->x >> 3;
1589     y_off += mv->y >> 3;
1590     src1  += y_off * src1stride + (x_off * (1 << s->ps.sps->pixel_shift));
1591     src2  += y_off * src2stride + (x_off * (1 << s->ps.sps->pixel_shift));
1592
1593     if (x_off < EPEL_EXTRA_BEFORE || y_off < EPEL_EXTRA_AFTER ||
1594         x_off >= pic_width - block_w - EPEL_EXTRA_AFTER ||
1595         y_off >= pic_height - block_h - EPEL_EXTRA_AFTER) {
1596         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1597         int offset1 = EPEL_EXTRA_BEFORE * (src1stride + (1 << s->ps.sps->pixel_shift));
1598         int buf_offset1 = EPEL_EXTRA_BEFORE *
1599                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1600         int offset2 = EPEL_EXTRA_BEFORE * (src2stride + (1 << s->ps.sps->pixel_shift));
1601         int buf_offset2 = EPEL_EXTRA_BEFORE *
1602                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1603
1604         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src1 - offset1,
1605                                  edge_emu_stride, src1stride,
1606                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1607                                  x_off - EPEL_EXTRA_BEFORE,
1608                                  y_off - EPEL_EXTRA_BEFORE,
1609                                  pic_width, pic_height);
1610
1611         src1 = lc->edge_emu_buffer + buf_offset1;
1612         src1stride = edge_emu_stride;
1613         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1614                                                        block_h, mx, my, lc->mc_buffer);
1615
1616         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src2 - offset2,
1617                                  edge_emu_stride, src2stride,
1618                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1619                                  x_off - EPEL_EXTRA_BEFORE,
1620                                  y_off - EPEL_EXTRA_BEFORE,
1621                                  pic_width, pic_height);
1622         src2 = lc->edge_emu_buffer + buf_offset2;
1623         src2stride = edge_emu_stride;
1624
1625         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1626                                                        block_h, mx, my, lc->mc_buffer);
1627     } else {
1628         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1629                                                        block_h, mx, my, lc->mc_buffer);
1630         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1631                                                        block_h, mx, my, lc->mc_buffer);
1632     }
1633 }
1634
1635 static void hevc_await_progress(HEVCContext *s, HEVCFrame *ref,
1636                                 const Mv *mv, int y0, int height)
1637 {
1638     int y = (mv->y >> 2) + y0 + height + 9;
1639     ff_thread_await_progress(&ref->tf, y, 0);
1640 }
1641
1642 static void hevc_luma_mv_mpv_mode(HEVCContext *s, int x0, int y0, int nPbW,
1643                                   int nPbH, int log2_cb_size, int part_idx,
1644                                   int merge_idx, MvField *mv)
1645 {
1646     HEVCLocalContext *lc             = &s->HEVClc;
1647     enum InterPredIdc inter_pred_idc = PRED_L0;
1648     int mvp_flag;
1649
1650     ff_hevc_set_neighbour_available(s, x0, y0, nPbW, nPbH);
1651     if (s->sh.slice_type == HEVC_SLICE_B)
1652         inter_pred_idc = ff_hevc_inter_pred_idc_decode(s, nPbW, nPbH);
1653
1654     if (inter_pred_idc != PRED_L1) {
1655         if (s->sh.nb_refs[L0])
1656             mv->ref_idx[0]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L0]);
1657
1658         mv->pred_flag[0] = 1;
1659         hls_mvd_coding(s, x0, y0, 0);
1660         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1661         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1662                                  part_idx, merge_idx, mv, mvp_flag, 0);
1663         mv->mv[0].x += lc->pu.mvd.x;
1664         mv->mv[0].y += lc->pu.mvd.y;
1665     }
1666
1667     if (inter_pred_idc != PRED_L0) {
1668         if (s->sh.nb_refs[L1])
1669             mv->ref_idx[1]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L1]);
1670
1671         if (s->sh.mvd_l1_zero_flag == 1 && inter_pred_idc == PRED_BI) {
1672             AV_ZERO32(&lc->pu.mvd);
1673         } else {
1674             hls_mvd_coding(s, x0, y0, 1);
1675         }
1676
1677         mv->pred_flag[1] = 1;
1678         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1679         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1680                                  part_idx, merge_idx, mv, mvp_flag, 1);
1681         mv->mv[1].x += lc->pu.mvd.x;
1682         mv->mv[1].y += lc->pu.mvd.y;
1683     }
1684 }
1685
1686 static void hls_prediction_unit(HEVCContext *s, int x0, int y0,
1687                                 int nPbW, int nPbH,
1688                                 int log2_cb_size, int partIdx)
1689 {
1690     static const int pred_indices[] = {
1691         [4] = 0, [8] = 1, [12] = 2, [16] = 3, [24] = 4, [32] = 5, [48] = 6, [64] = 7,
1692     };
1693     const int pred_idx = pred_indices[nPbW];
1694
1695 #define POS(c_idx, x, y)                                                              \
1696     &s->frame->data[c_idx][((y) >> s->ps.sps->vshift[c_idx]) * s->frame->linesize[c_idx] + \
1697                            (((x) >> s->ps.sps->hshift[c_idx]) << s->ps.sps->pixel_shift)]
1698     HEVCLocalContext *lc = &s->HEVClc;
1699     int merge_idx = 0;
1700     struct MvField current_mv = {{{ 0 }}};
1701
1702     int min_pu_width = s->ps.sps->min_pu_width;
1703     int weighted_pred = (s->sh.slice_type == HEVC_SLICE_P && s->ps.pps->weighted_pred_flag) ||
1704                         (s->sh.slice_type == HEVC_SLICE_B && s->ps.pps->weighted_bipred_flag);
1705
1706     MvField *tab_mvf = s->ref->tab_mvf;
1707     RefPicList  *refPicList = s->ref->refPicList;
1708     HEVCFrame *ref0, *ref1;
1709
1710     ptrdiff_t tmpstride = MAX_PB_SIZE * sizeof(int16_t);
1711
1712     uint8_t *dst0 = POS(0, x0, y0);
1713     uint8_t *dst1 = POS(1, x0, y0);
1714     uint8_t *dst2 = POS(2, x0, y0);
1715     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
1716     int min_cb_width     = s->ps.sps->min_cb_width;
1717     int x_cb             = x0 >> log2_min_cb_size;
1718     int y_cb             = y0 >> log2_min_cb_size;
1719     int x_pu, y_pu;
1720     int i, j;
1721
1722     int skip_flag = SAMPLE_CTB(s->skip_flag, x_cb, y_cb);
1723
1724     if (!skip_flag)
1725         lc->pu.merge_flag = ff_hevc_merge_flag_decode(s);
1726
1727     if (skip_flag || lc->pu.merge_flag) {
1728         if (s->sh.max_num_merge_cand > 1)
1729             merge_idx = ff_hevc_merge_idx_decode(s);
1730         else
1731             merge_idx = 0;
1732
1733         ff_hevc_luma_mv_merge_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1734                                    partIdx, merge_idx, &current_mv);
1735     } else {
1736         hevc_luma_mv_mpv_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1737                               partIdx, merge_idx, &current_mv);
1738     }
1739
1740     x_pu = x0 >> s->ps.sps->log2_min_pu_size;
1741     y_pu = y0 >> s->ps.sps->log2_min_pu_size;
1742
1743     for (j = 0; j < nPbH >> s->ps.sps->log2_min_pu_size; j++)
1744         for (i = 0; i < nPbW >> s->ps.sps->log2_min_pu_size; i++)
1745             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i] = current_mv;
1746
1747     if (current_mv.pred_flag[0]) {
1748         ref0 = refPicList[0].ref[current_mv.ref_idx[0]];
1749         if (!ref0)
1750             return;
1751         hevc_await_progress(s, ref0, &current_mv.mv[0], y0, nPbH);
1752     }
1753     if (current_mv.pred_flag[1]) {
1754         ref1 = refPicList[1].ref[current_mv.ref_idx[1]];
1755         if (!ref1)
1756             return;
1757         hevc_await_progress(s, ref1, &current_mv.mv[1], y0, nPbH);
1758     }
1759
1760     if (current_mv.pred_flag[0] && !current_mv.pred_flag[1]) {
1761         LOCAL_ALIGNED_16(int16_t,  tmp, [MAX_PB_SIZE * MAX_PB_SIZE]);
1762         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1763
1764         luma_mc(s, tmp, tmpstride, ref0->frame,
1765                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1766
1767         if (weighted_pred) {
1768             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1769                                                s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1770                                                s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1771                                                dst0, s->frame->linesize[0], tmp,
1772                                                tmpstride, nPbH);
1773         } else {
1774             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1775         }
1776         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1777                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1778
1779         if (weighted_pred) {
1780             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1781                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1782                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1783                                                       dst1, s->frame->linesize[1], tmp, tmpstride,
1784                                                       nPbH / 2);
1785             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1786                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1787                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1788                                                       dst2, s->frame->linesize[2], tmp2, tmpstride,
1789                                                       nPbH / 2);
1790         } else {
1791             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1792             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1793         }
1794     } else if (!current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1795         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1796         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1797
1798         luma_mc(s, tmp, tmpstride, ref1->frame,
1799                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1800
1801         if (weighted_pred) {
1802             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1803                                                s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1804                                                s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1805                                                dst0, s->frame->linesize[0], tmp, tmpstride,
1806                                                nPbH);
1807         } else {
1808             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1809         }
1810
1811         chroma_mc(s, tmp, tmp2, tmpstride, ref1->frame,
1812                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1813
1814         if (weighted_pred) {
1815             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1816                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1817                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1818                                                       dst1, s->frame->linesize[1], tmp, tmpstride, nPbH/2);
1819             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1820                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1821                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1822                                                       dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH/2);
1823         } else {
1824             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1825             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1826         }
1827     } else if (current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1828         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1829         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1830         LOCAL_ALIGNED_16(int16_t, tmp3, [MAX_PB_SIZE * MAX_PB_SIZE]);
1831         LOCAL_ALIGNED_16(int16_t, tmp4, [MAX_PB_SIZE * MAX_PB_SIZE]);
1832
1833         luma_mc(s, tmp, tmpstride, ref0->frame,
1834                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1835         luma_mc(s, tmp2, tmpstride, ref1->frame,
1836                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1837
1838         if (weighted_pred) {
1839             s->hevcdsp.weighted_pred_avg[pred_idx](s->sh.luma_log2_weight_denom,
1840                                                    s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1841                                                    s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1842                                                    s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1843                                                    s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1844                                                    dst0, s->frame->linesize[0],
1845                                                    tmp, tmp2, tmpstride, nPbH);
1846         } else {
1847             s->hevcdsp.put_unweighted_pred_avg[pred_idx](dst0, s->frame->linesize[0],
1848                                                          tmp, tmp2, tmpstride, nPbH);
1849         }
1850
1851         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1852                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1853         chroma_mc(s, tmp3, tmp4, tmpstride, ref1->frame,
1854                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1855
1856         if (weighted_pred) {
1857             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1858                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1859                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1860                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1861                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1862                                                           dst1, s->frame->linesize[1], tmp, tmp3,
1863                                                           tmpstride, nPbH / 2);
1864             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1865                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1866                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1867                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1868                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1869                                                           dst2, s->frame->linesize[2], tmp2, tmp4,
1870                                                           tmpstride, nPbH / 2);
1871         } else {
1872             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst1, s->frame->linesize[1], tmp, tmp3,  tmpstride, nPbH/2);
1873             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmp4, tmpstride, nPbH/2);
1874         }
1875     }
1876 }
1877
1878 /**
1879  * 8.4.1
1880  */
1881 static int luma_intra_pred_mode(HEVCContext *s, int x0, int y0, int pu_size,
1882                                 int prev_intra_luma_pred_flag)
1883 {
1884     HEVCLocalContext *lc = &s->HEVClc;
1885     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
1886     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
1887     int min_pu_width     = s->ps.sps->min_pu_width;
1888     int size_in_pus      = pu_size >> s->ps.sps->log2_min_pu_size;
1889     int x0b              = x0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1890     int y0b              = y0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1891
1892     int cand_up   = (lc->ctb_up_flag || y0b) ?
1893                     s->tab_ipm[(y_pu - 1) * min_pu_width + x_pu] : INTRA_DC;
1894     int cand_left = (lc->ctb_left_flag || x0b) ?
1895                     s->tab_ipm[y_pu * min_pu_width + x_pu - 1]   : INTRA_DC;
1896
1897     int y_ctb = (y0 >> (s->ps.sps->log2_ctb_size)) << (s->ps.sps->log2_ctb_size);
1898
1899     MvField *tab_mvf = s->ref->tab_mvf;
1900     int intra_pred_mode;
1901     int candidate[3];
1902     int i, j;
1903
1904     // intra_pred_mode prediction does not cross vertical CTB boundaries
1905     if ((y0 - 1) < y_ctb)
1906         cand_up = INTRA_DC;
1907
1908     if (cand_left == cand_up) {
1909         if (cand_left < 2) {
1910             candidate[0] = INTRA_PLANAR;
1911             candidate[1] = INTRA_DC;
1912             candidate[2] = INTRA_ANGULAR_26;
1913         } else {
1914             candidate[0] = cand_left;
1915             candidate[1] = 2 + ((cand_left - 2 - 1 + 32) & 31);
1916             candidate[2] = 2 + ((cand_left - 2 + 1) & 31);
1917         }
1918     } else {
1919         candidate[0] = cand_left;
1920         candidate[1] = cand_up;
1921         if (candidate[0] != INTRA_PLANAR && candidate[1] != INTRA_PLANAR) {
1922             candidate[2] = INTRA_PLANAR;
1923         } else if (candidate[0] != INTRA_DC && candidate[1] != INTRA_DC) {
1924             candidate[2] = INTRA_DC;
1925         } else {
1926             candidate[2] = INTRA_ANGULAR_26;
1927         }
1928     }
1929
1930     if (prev_intra_luma_pred_flag) {
1931         intra_pred_mode = candidate[lc->pu.mpm_idx];
1932     } else {
1933         if (candidate[0] > candidate[1])
1934             FFSWAP(uint8_t, candidate[0], candidate[1]);
1935         if (candidate[0] > candidate[2])
1936             FFSWAP(uint8_t, candidate[0], candidate[2]);
1937         if (candidate[1] > candidate[2])
1938             FFSWAP(uint8_t, candidate[1], candidate[2]);
1939
1940         intra_pred_mode = lc->pu.rem_intra_luma_pred_mode;
1941         for (i = 0; i < 3; i++)
1942             if (intra_pred_mode >= candidate[i])
1943                 intra_pred_mode++;
1944     }
1945
1946     /* write the intra prediction units into the mv array */
1947     if (!size_in_pus)
1948         size_in_pus = 1;
1949     for (i = 0; i < size_in_pus; i++) {
1950         memset(&s->tab_ipm[(y_pu + i) * min_pu_width + x_pu],
1951                intra_pred_mode, size_in_pus);
1952
1953         for (j = 0; j < size_in_pus; j++) {
1954             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].is_intra     = 1;
1955             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[0] = 0;
1956             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[1] = 0;
1957             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[0]   = 0;
1958             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[1]   = 0;
1959             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].x      = 0;
1960             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].y      = 0;
1961             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].x      = 0;
1962             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].y      = 0;
1963         }
1964     }
1965
1966     return intra_pred_mode;
1967 }
1968
1969 static av_always_inline void set_ct_depth(HEVCContext *s, int x0, int y0,
1970                                           int log2_cb_size, int ct_depth)
1971 {
1972     int length = (1 << log2_cb_size) >> s->ps.sps->log2_min_cb_size;
1973     int x_cb   = x0 >> s->ps.sps->log2_min_cb_size;
1974     int y_cb   = y0 >> s->ps.sps->log2_min_cb_size;
1975     int y;
1976
1977     for (y = 0; y < length; y++)
1978         memset(&s->tab_ct_depth[(y_cb + y) * s->ps.sps->min_cb_width + x_cb],
1979                ct_depth, length);
1980 }
1981
1982 static void intra_prediction_unit(HEVCContext *s, int x0, int y0,
1983                                   int log2_cb_size)
1984 {
1985     HEVCLocalContext *lc = &s->HEVClc;
1986     static const uint8_t intra_chroma_table[4] = { 0, 26, 10, 1 };
1987     uint8_t prev_intra_luma_pred_flag[4];
1988     int split   = lc->cu.part_mode == PART_NxN;
1989     int pb_size = (1 << log2_cb_size) >> split;
1990     int side    = split + 1;
1991     int chroma_mode;
1992     int i, j;
1993
1994     for (i = 0; i < side; i++)
1995         for (j = 0; j < side; j++)
1996             prev_intra_luma_pred_flag[2 * i + j] = ff_hevc_prev_intra_luma_pred_flag_decode(s);
1997
1998     for (i = 0; i < side; i++) {
1999         for (j = 0; j < side; j++) {
2000             if (prev_intra_luma_pred_flag[2 * i + j])
2001                 lc->pu.mpm_idx = ff_hevc_mpm_idx_decode(s);
2002             else
2003                 lc->pu.rem_intra_luma_pred_mode = ff_hevc_rem_intra_luma_pred_mode_decode(s);
2004
2005             lc->pu.intra_pred_mode[2 * i + j] =
2006                 luma_intra_pred_mode(s, x0 + pb_size * j, y0 + pb_size * i, pb_size,
2007                                      prev_intra_luma_pred_flag[2 * i + j]);
2008         }
2009     }
2010
2011     chroma_mode = ff_hevc_intra_chroma_pred_mode_decode(s);
2012     if (chroma_mode != 4) {
2013         if (lc->pu.intra_pred_mode[0] == intra_chroma_table[chroma_mode])
2014             lc->pu.intra_pred_mode_c = 34;
2015         else
2016             lc->pu.intra_pred_mode_c = intra_chroma_table[chroma_mode];
2017     } else {
2018         lc->pu.intra_pred_mode_c = lc->pu.intra_pred_mode[0];
2019     }
2020 }
2021
2022 static void intra_prediction_unit_default_value(HEVCContext *s,
2023                                                 int x0, int y0,
2024                                                 int log2_cb_size)
2025 {
2026     HEVCLocalContext *lc = &s->HEVClc;
2027     int pb_size          = 1 << log2_cb_size;
2028     int size_in_pus      = pb_size >> s->ps.sps->log2_min_pu_size;
2029     int min_pu_width     = s->ps.sps->min_pu_width;
2030     MvField *tab_mvf     = s->ref->tab_mvf;
2031     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
2032     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
2033     int j, k;
2034
2035     if (size_in_pus == 0)
2036         size_in_pus = 1;
2037     for (j = 0; j < size_in_pus; j++) {
2038         memset(&s->tab_ipm[(y_pu + j) * min_pu_width + x_pu], INTRA_DC, size_in_pus);
2039         for (k = 0; k < size_in_pus; k++)
2040             tab_mvf[(y_pu + j) * min_pu_width + x_pu + k].is_intra = lc->cu.pred_mode == MODE_INTRA;
2041     }
2042 }
2043
2044 static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
2045 {
2046     int cb_size          = 1 << log2_cb_size;
2047     HEVCLocalContext *lc = &s->HEVClc;
2048     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
2049     int length           = cb_size >> log2_min_cb_size;
2050     int min_cb_width     = s->ps.sps->min_cb_width;
2051     int x_cb             = x0 >> log2_min_cb_size;
2052     int y_cb             = y0 >> log2_min_cb_size;
2053     int x, y, ret;
2054
2055     lc->cu.x                = x0;
2056     lc->cu.y                = y0;
2057     lc->cu.pred_mode        = MODE_INTRA;
2058     lc->cu.part_mode        = PART_2Nx2N;
2059     lc->cu.intra_split_flag = 0;
2060
2061     SAMPLE_CTB(s->skip_flag, x_cb, y_cb) = 0;
2062     for (x = 0; x < 4; x++)
2063         lc->pu.intra_pred_mode[x] = 1;
2064     if (s->ps.pps->transquant_bypass_enable_flag) {
2065         lc->cu.cu_transquant_bypass_flag = ff_hevc_cu_transquant_bypass_flag_decode(s);
2066         if (lc->cu.cu_transquant_bypass_flag)
2067             set_deblocking_bypass(s, x0, y0, log2_cb_size);
2068     } else
2069         lc->cu.cu_transquant_bypass_flag = 0;
2070
2071     if (s->sh.slice_type != HEVC_SLICE_I) {
2072         uint8_t skip_flag = ff_hevc_skip_flag_decode(s, x0, y0, x_cb, y_cb);
2073
2074         x = y_cb * min_cb_width + x_cb;
2075         for (y = 0; y < length; y++) {
2076             memset(&s->skip_flag[x], skip_flag, length);
2077             x += min_cb_width;
2078         }
2079         lc->cu.pred_mode = skip_flag ? MODE_SKIP : MODE_INTER;
2080     }
2081
2082     if (SAMPLE_CTB(s->skip_flag, x_cb, y_cb)) {
2083         hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2084         intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2085
2086         if (!s->sh.disable_deblocking_filter_flag)
2087             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2088     } else {
2089         int pcm_flag = 0;
2090
2091         if (s->sh.slice_type != HEVC_SLICE_I)
2092             lc->cu.pred_mode = ff_hevc_pred_mode_decode(s);
2093         if (lc->cu.pred_mode != MODE_INTRA ||
2094             log2_cb_size == s->ps.sps->log2_min_cb_size) {
2095             lc->cu.part_mode        = ff_hevc_part_mode_decode(s, log2_cb_size);
2096             lc->cu.intra_split_flag = lc->cu.part_mode == PART_NxN &&
2097                                       lc->cu.pred_mode == MODE_INTRA;
2098         }
2099
2100         if (lc->cu.pred_mode == MODE_INTRA) {
2101             if (lc->cu.part_mode == PART_2Nx2N && s->ps.sps->pcm_enabled_flag &&
2102                 log2_cb_size >= s->ps.sps->pcm.log2_min_pcm_cb_size &&
2103                 log2_cb_size <= s->ps.sps->pcm.log2_max_pcm_cb_size) {
2104                 pcm_flag = ff_hevc_pcm_flag_decode(s);
2105             }
2106             if (pcm_flag) {
2107                 intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2108                 ret = hls_pcm_sample(s, x0, y0, log2_cb_size);
2109                 if (s->ps.sps->pcm.loop_filter_disable_flag)
2110                     set_deblocking_bypass(s, x0, y0, log2_cb_size);
2111
2112                 if (ret < 0)
2113                     return ret;
2114             } else {
2115                 intra_prediction_unit(s, x0, y0, log2_cb_size);
2116             }
2117         } else {
2118             intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2119             switch (lc->cu.part_mode) {
2120             case PART_2Nx2N:
2121                 hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2122                 break;
2123             case PART_2NxN:
2124                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size / 2, log2_cb_size, 0);
2125                 hls_prediction_unit(s, x0, y0 + cb_size / 2, cb_size, cb_size / 2, log2_cb_size, 1);
2126                 break;
2127             case PART_Nx2N:
2128                 hls_prediction_unit(s, x0,               y0, cb_size / 2, cb_size, log2_cb_size, 0);
2129                 hls_prediction_unit(s, x0 + cb_size / 2, y0, cb_size / 2, cb_size, log2_cb_size, 1);
2130                 break;
2131             case PART_2NxnU:
2132                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size     / 4, log2_cb_size, 0);
2133                 hls_prediction_unit(s, x0, y0 + cb_size / 4, cb_size, cb_size * 3 / 4, log2_cb_size, 1);
2134                 break;
2135             case PART_2NxnD:
2136                 hls_prediction_unit(s, x0, y0,                   cb_size, cb_size * 3 / 4, log2_cb_size, 0);
2137                 hls_prediction_unit(s, x0, y0 + cb_size * 3 / 4, cb_size, cb_size     / 4, log2_cb_size, 1);
2138                 break;
2139             case PART_nLx2N:
2140                 hls_prediction_unit(s, x0,               y0, cb_size     / 4, cb_size, log2_cb_size, 0);
2141                 hls_prediction_unit(s, x0 + cb_size / 4, y0, cb_size * 3 / 4, cb_size, log2_cb_size, 1);
2142                 break;
2143             case PART_nRx2N:
2144                 hls_prediction_unit(s, x0,                   y0, cb_size * 3 / 4, cb_size, log2_cb_size, 0);
2145                 hls_prediction_unit(s, x0 + cb_size * 3 / 4, y0, cb_size     / 4, cb_size, log2_cb_size, 1);
2146                 break;
2147             case PART_NxN:
2148                 hls_prediction_unit(s, x0,               y0,               cb_size / 2, cb_size / 2, log2_cb_size, 0);
2149                 hls_prediction_unit(s, x0 + cb_size / 2, y0,               cb_size / 2, cb_size / 2, log2_cb_size, 1);
2150                 hls_prediction_unit(s, x0,               y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 2);
2151                 hls_prediction_unit(s, x0 + cb_size / 2, y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 3);
2152                 break;
2153             }
2154         }
2155
2156         if (!pcm_flag) {
2157             int rqt_root_cbf = 1;
2158
2159             if (lc->cu.pred_mode != MODE_INTRA &&
2160                 !(lc->cu.part_mode == PART_2Nx2N && lc->pu.merge_flag)) {
2161                 rqt_root_cbf = ff_hevc_no_residual_syntax_flag_decode(s);
2162             }
2163             if (rqt_root_cbf) {
2164                 lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ?
2165                                          s->ps.sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag :
2166                                          s->ps.sps->max_transform_hierarchy_depth_inter;
2167                 ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0,
2168                                          log2_cb_size,
2169                                          log2_cb_size, 0, 0, 0, 0);
2170                 if (ret < 0)
2171                     return ret;
2172             } else {
2173                 if (!s->sh.disable_deblocking_filter_flag)
2174                     ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2175             }
2176         }
2177     }
2178
2179     if (s->ps.pps->cu_qp_delta_enabled_flag && lc->tu.is_cu_qp_delta_coded == 0)
2180         ff_hevc_set_qPy(s, x0, y0, x0, y0, log2_cb_size);
2181
2182     x = y_cb * min_cb_width + x_cb;
2183     for (y = 0; y < length; y++) {
2184         memset(&s->qp_y_tab[x], lc->qp_y, length);
2185         x += min_cb_width;
2186     }
2187
2188     set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth);
2189
2190     return 0;
2191 }
2192
2193 static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
2194                                int log2_cb_size, int cb_depth)
2195 {
2196     HEVCLocalContext *lc = &s->HEVClc;
2197     const int cb_size    = 1 << log2_cb_size;
2198     int split_cu;
2199
2200     lc->ct.depth = cb_depth;
2201     if (x0 + cb_size <= s->ps.sps->width  &&
2202         y0 + cb_size <= s->ps.sps->height &&
2203         log2_cb_size > s->ps.sps->log2_min_cb_size) {
2204         split_cu = ff_hevc_split_coding_unit_flag_decode(s, cb_depth, x0, y0);
2205     } else {
2206         split_cu = (log2_cb_size > s->ps.sps->log2_min_cb_size);
2207     }
2208     if (s->ps.pps->cu_qp_delta_enabled_flag &&
2209         log2_cb_size >= s->ps.sps->log2_ctb_size - s->ps.pps->diff_cu_qp_delta_depth) {
2210         lc->tu.is_cu_qp_delta_coded = 0;
2211         lc->tu.cu_qp_delta          = 0;
2212     }
2213
2214     if (split_cu) {
2215         const int cb_size_split = cb_size >> 1;
2216         const int x1 = x0 + cb_size_split;
2217         const int y1 = y0 + cb_size_split;
2218
2219         log2_cb_size--;
2220         cb_depth++;
2221
2222 #define SUBDIVIDE(x, y)                                                \
2223 do {                                                                   \
2224     if (x < s->ps.sps->width && y < s->ps.sps->height) {                     \
2225         int ret = hls_coding_quadtree(s, x, y, log2_cb_size, cb_depth);\
2226         if (ret < 0)                                                   \
2227             return ret;                                                \
2228     }                                                                  \
2229 } while (0)
2230
2231         SUBDIVIDE(x0, y0);
2232         SUBDIVIDE(x1, y0);
2233         SUBDIVIDE(x0, y1);
2234         SUBDIVIDE(x1, y1);
2235     } else {
2236         int ret = hls_coding_unit(s, x0, y0, log2_cb_size);
2237         if (ret < 0)
2238             return ret;
2239     }
2240
2241     return 0;
2242 }
2243
2244 static void hls_decode_neighbour(HEVCContext *s, int x_ctb, int y_ctb,
2245                                  int ctb_addr_ts)
2246 {
2247     HEVCLocalContext *lc  = &s->HEVClc;
2248     int ctb_size          = 1 << s->ps.sps->log2_ctb_size;
2249     int ctb_addr_rs       = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2250     int ctb_addr_in_slice = ctb_addr_rs - s->sh.slice_addr;
2251
2252     s->tab_slice_address[ctb_addr_rs] = s->sh.slice_addr;
2253
2254     if (s->ps.pps->entropy_coding_sync_enabled_flag) {
2255         if (x_ctb == 0 && (y_ctb & (ctb_size - 1)) == 0)
2256             lc->first_qp_group = 1;
2257         lc->end_of_tiles_x = s->ps.sps->width;
2258     } else if (s->ps.pps->tiles_enabled_flag) {
2259         if (ctb_addr_ts && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[ctb_addr_ts - 1]) {
2260             int idxX = s->ps.pps->col_idxX[x_ctb >> s->ps.sps->log2_ctb_size];
2261             lc->start_of_tiles_x = x_ctb;
2262             lc->end_of_tiles_x   = x_ctb + (s->ps.pps->column_width[idxX] << s->ps.sps->log2_ctb_size);
2263             lc->first_qp_group   = 1;
2264         }
2265     } else {
2266         lc->end_of_tiles_x = s->ps.sps->width;
2267     }
2268
2269     lc->end_of_tiles_y = FFMIN(y_ctb + ctb_size, s->ps.sps->height);
2270
2271     lc->boundary_flags = 0;
2272     if (s->ps.pps->tiles_enabled_flag) {
2273         if (x_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - 1]])
2274             lc->boundary_flags |= BOUNDARY_LEFT_TILE;
2275         if (x_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - 1])
2276             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2277         if (y_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - s->ps.sps->ctb_width]])
2278             lc->boundary_flags |= BOUNDARY_UPPER_TILE;
2279         if (y_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - s->ps.sps->ctb_width])
2280             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2281     } else {
2282         if (!ctb_addr_in_slice)
2283             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2284         if (ctb_addr_in_slice < s->ps.sps->ctb_width)
2285             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2286     }
2287
2288     lc->ctb_left_flag = ((x_ctb > 0) && (ctb_addr_in_slice > 0) && !(lc->boundary_flags & BOUNDARY_LEFT_TILE));
2289     lc->ctb_up_flag   = ((y_ctb > 0) && (ctb_addr_in_slice >= s->ps.sps->ctb_width) && !(lc->boundary_flags & BOUNDARY_UPPER_TILE));
2290     lc->ctb_up_right_flag = ((y_ctb > 0)  && (ctb_addr_in_slice+1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs+1 - s->ps.sps->ctb_width]]));
2291     lc->ctb_up_left_flag = ((x_ctb > 0) && (y_ctb > 0)  && (ctb_addr_in_slice-1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs-1 - s->ps.sps->ctb_width]]));
2292 }
2293
2294 static int hls_slice_data(HEVCContext *s)
2295 {
2296     int ctb_size    = 1 << s->ps.sps->log2_ctb_size;
2297     int more_data   = 1;
2298     int x_ctb       = 0;
2299     int y_ctb       = 0;
2300     int ctb_addr_ts = s->ps.pps->ctb_addr_rs_to_ts[s->sh.slice_ctb_addr_rs];
2301     int ret;
2302
2303     while (more_data && ctb_addr_ts < s->ps.sps->ctb_size) {
2304         int ctb_addr_rs = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2305
2306         x_ctb = (ctb_addr_rs % ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2307         y_ctb = (ctb_addr_rs / ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2308         hls_decode_neighbour(s, x_ctb, y_ctb, ctb_addr_ts);
2309
2310         ff_hevc_cabac_init(s, ctb_addr_ts);
2311
2312         hls_sao_param(s, x_ctb >> s->ps.sps->log2_ctb_size, y_ctb >> s->ps.sps->log2_ctb_size);
2313
2314         s->deblock[ctb_addr_rs].beta_offset = s->sh.beta_offset;
2315         s->deblock[ctb_addr_rs].tc_offset   = s->sh.tc_offset;
2316         s->filter_slice_edges[ctb_addr_rs]  = s->sh.slice_loop_filter_across_slices_enabled_flag;
2317
2318         ret = hls_coding_quadtree(s, x_ctb, y_ctb, s->ps.sps->log2_ctb_size, 0);
2319         if (ret < 0)
2320             return ret;
2321         more_data = !ff_hevc_end_of_slice_flag_decode(s);
2322
2323         ctb_addr_ts++;
2324         ff_hevc_save_states(s, ctb_addr_ts);
2325         ff_hevc_hls_filters(s, x_ctb, y_ctb, ctb_size);
2326     }
2327
2328     if (x_ctb + ctb_size >= s->ps.sps->width &&
2329         y_ctb + ctb_size >= s->ps.sps->height)
2330         ff_hevc_hls_filter(s, x_ctb, y_ctb);
2331
2332     return ctb_addr_ts;
2333 }
2334
2335 static void restore_tqb_pixels(HEVCContext *s)
2336 {
2337     int min_pu_size = 1 << s->ps.sps->log2_min_pu_size;
2338     int x, y, c_idx;
2339
2340     for (c_idx = 0; c_idx < 3; c_idx++) {
2341         ptrdiff_t stride = s->frame->linesize[c_idx];
2342         int hshift       = s->ps.sps->hshift[c_idx];
2343         int vshift       = s->ps.sps->vshift[c_idx];
2344         for (y = 0; y < s->ps.sps->min_pu_height; y++) {
2345             for (x = 0; x < s->ps.sps->min_pu_width; x++) {
2346                 if (s->is_pcm[y * s->ps.sps->min_pu_width + x]) {
2347                     int n;
2348                     int len      = min_pu_size >> hshift;
2349                     uint8_t *src = &s->frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2350                     uint8_t *dst = &s->sao_frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2351                     for (n = 0; n < (min_pu_size >> vshift); n++) {
2352                         memcpy(dst, src, len);
2353                         src += stride;
2354                         dst += stride;
2355                     }
2356                 }
2357             }
2358         }
2359     }
2360 }
2361
2362 static int set_side_data(HEVCContext *s)
2363 {
2364     AVFrame *out = s->ref->frame;
2365
2366     if (s->sei.frame_packing.present &&
2367         s->sei.frame_packing.arrangement_type >= 3 &&
2368         s->sei.frame_packing.arrangement_type <= 5 &&
2369         s->sei.frame_packing.content_interpretation_type > 0 &&
2370         s->sei.frame_packing.content_interpretation_type < 3) {
2371         AVStereo3D *stereo = av_stereo3d_create_side_data(out);
2372         if (!stereo)
2373             return AVERROR(ENOMEM);
2374
2375         switch (s->sei.frame_packing.arrangement_type) {
2376         case 3:
2377             if (s->sei.frame_packing.quincunx_subsampling)
2378                 stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
2379             else
2380                 stereo->type = AV_STEREO3D_SIDEBYSIDE;
2381             break;
2382         case 4:
2383             stereo->type = AV_STEREO3D_TOPBOTTOM;
2384             break;
2385         case 5:
2386             stereo->type = AV_STEREO3D_FRAMESEQUENCE;
2387             break;
2388         }
2389
2390         if (s->sei.frame_packing.content_interpretation_type == 2)
2391             stereo->flags = AV_STEREO3D_FLAG_INVERT;
2392     }
2393
2394     if (s->sei.display_orientation.present &&
2395         (s->sei.display_orientation.anticlockwise_rotation ||
2396          s->sei.display_orientation.hflip || s->sei.display_orientation.vflip)) {
2397         double angle = s->sei.display_orientation.anticlockwise_rotation * 360 / (double) (1 << 16);
2398         AVFrameSideData *rotation = av_frame_new_side_data(out,
2399                                                            AV_FRAME_DATA_DISPLAYMATRIX,
2400                                                            sizeof(int32_t) * 9);
2401         if (!rotation)
2402             return AVERROR(ENOMEM);
2403
2404         av_display_rotation_set((int32_t *)rotation->data, angle);
2405         av_display_matrix_flip((int32_t *)rotation->data,
2406                                s->sei.display_orientation.hflip,
2407                                s->sei.display_orientation.vflip);
2408     }
2409
2410     return 0;
2411 }
2412
2413 static int hevc_frame_start(HEVCContext *s)
2414 {
2415     HEVCLocalContext *lc = &s->HEVClc;
2416     int ret;
2417
2418     memset(s->horizontal_bs, 0, 2 * s->bs_width * (s->bs_height + 1));
2419     memset(s->vertical_bs,   0, 2 * s->bs_width * (s->bs_height + 1));
2420     memset(s->cbf_luma,      0, s->ps.sps->min_tb_width * s->ps.sps->min_tb_height);
2421     memset(s->is_pcm,        0, s->ps.sps->min_pu_width * s->ps.sps->min_pu_height);
2422
2423     lc->start_of_tiles_x = 0;
2424     s->is_decoded        = 0;
2425     s->first_nal_type    = s->nal_unit_type;
2426
2427     if (s->ps.pps->tiles_enabled_flag)
2428         lc->end_of_tiles_x = s->ps.pps->column_width[0] << s->ps.sps->log2_ctb_size;
2429
2430     ret = ff_hevc_set_new_ref(s, s->ps.sps->sao_enabled ? &s->sao_frame : &s->frame,
2431                               s->poc);
2432     if (ret < 0)
2433         goto fail;
2434
2435     ret = ff_hevc_frame_rps(s);
2436     if (ret < 0) {
2437         av_log(s->avctx, AV_LOG_ERROR, "Error constructing the frame RPS.\n");
2438         goto fail;
2439     }
2440
2441     s->ref->frame->key_frame = IS_IRAP(s);
2442
2443     ret = set_side_data(s);
2444     if (ret < 0)
2445         goto fail;
2446
2447     av_frame_unref(s->output_frame);
2448     ret = ff_hevc_output_frame(s, s->output_frame, 0);
2449     if (ret < 0)
2450         goto fail;
2451
2452     ff_thread_finish_setup(s->avctx);
2453
2454     return 0;
2455
2456 fail:
2457     if (s->ref)
2458         ff_hevc_unref_frame(s, s->ref, ~0);
2459     s->ref = NULL;
2460     return ret;
2461 }
2462
2463 static int decode_nal_unit(HEVCContext *s, const H2645NAL *nal)
2464 {
2465     HEVCLocalContext *lc = &s->HEVClc;
2466     GetBitContext *gb    = &lc->gb;
2467     int ctb_addr_ts, ret;
2468
2469     *gb              = nal->gb;
2470     s->nal_unit_type = nal->type;
2471     s->temporal_id   = nal->temporal_id;
2472
2473     switch (s->nal_unit_type) {
2474     case HEVC_NAL_VPS:
2475         ret = ff_hevc_decode_nal_vps(gb, s->avctx, &s->ps);
2476         if (ret < 0)
2477             goto fail;
2478         break;
2479     case HEVC_NAL_SPS:
2480         ret = ff_hevc_decode_nal_sps(gb, s->avctx, &s->ps,
2481                                      s->apply_defdispwin);
2482         if (ret < 0)
2483             goto fail;
2484         break;
2485     case HEVC_NAL_PPS:
2486         ret = ff_hevc_decode_nal_pps(gb, s->avctx, &s->ps);
2487         if (ret < 0)
2488             goto fail;
2489         break;
2490     case HEVC_NAL_SEI_PREFIX:
2491     case HEVC_NAL_SEI_SUFFIX:
2492         ret = ff_hevc_decode_nal_sei(gb, s->avctx, &s->sei,
2493                                      s->nal_unit_type);
2494         if (ret < 0)
2495             goto fail;
2496         break;
2497     case HEVC_NAL_TRAIL_R:
2498     case HEVC_NAL_TRAIL_N:
2499     case HEVC_NAL_TSA_N:
2500     case HEVC_NAL_TSA_R:
2501     case HEVC_NAL_STSA_N:
2502     case HEVC_NAL_STSA_R:
2503     case HEVC_NAL_BLA_W_LP:
2504     case HEVC_NAL_BLA_W_RADL:
2505     case HEVC_NAL_BLA_N_LP:
2506     case HEVC_NAL_IDR_W_RADL:
2507     case HEVC_NAL_IDR_N_LP:
2508     case HEVC_NAL_CRA_NUT:
2509     case HEVC_NAL_RADL_N:
2510     case HEVC_NAL_RADL_R:
2511     case HEVC_NAL_RASL_N:
2512     case HEVC_NAL_RASL_R:
2513         ret = hls_slice_header(s);
2514         if (ret < 0)
2515             return ret;
2516
2517         if (s->max_ra == INT_MAX) {
2518             if (s->nal_unit_type == HEVC_NAL_CRA_NUT || IS_BLA(s)) {
2519                 s->max_ra = s->poc;
2520             } else {
2521                 if (IS_IDR(s))
2522                     s->max_ra = INT_MIN;
2523             }
2524         }
2525
2526         if ((s->nal_unit_type == HEVC_NAL_RASL_R || s->nal_unit_type == HEVC_NAL_RASL_N) &&
2527             s->poc <= s->max_ra) {
2528             s->is_decoded = 0;
2529             break;
2530         } else {
2531             if (s->nal_unit_type == HEVC_NAL_RASL_R && s->poc > s->max_ra)
2532                 s->max_ra = INT_MIN;
2533         }
2534
2535         if (s->sh.first_slice_in_pic_flag) {
2536             ret = hevc_frame_start(s);
2537             if (ret < 0)
2538                 return ret;
2539         } else if (!s->ref) {
2540             av_log(s->avctx, AV_LOG_ERROR, "First slice in a frame missing.\n");
2541             goto fail;
2542         }
2543
2544         if (s->nal_unit_type != s->first_nal_type) {
2545             av_log(s->avctx, AV_LOG_ERROR,
2546                    "Non-matching NAL types of the VCL NALUs: %d %d\n",
2547                    s->first_nal_type, s->nal_unit_type);
2548             return AVERROR_INVALIDDATA;
2549         }
2550
2551         if (!s->sh.dependent_slice_segment_flag &&
2552             s->sh.slice_type != HEVC_SLICE_I) {
2553             ret = ff_hevc_slice_rpl(s);
2554             if (ret < 0) {
2555                 av_log(s->avctx, AV_LOG_WARNING,
2556                        "Error constructing the reference lists for the current slice.\n");
2557                 goto fail;
2558             }
2559         }
2560
2561         if (s->sh.first_slice_in_pic_flag && s->avctx->hwaccel) {
2562             ret = s->avctx->hwaccel->start_frame(s->avctx, NULL, 0);
2563             if (ret < 0)
2564                 goto fail;
2565         }
2566
2567         if (s->avctx->hwaccel) {
2568             ret = s->avctx->hwaccel->decode_slice(s->avctx, nal->raw_data, nal->raw_size);
2569             if (ret < 0)
2570                 goto fail;
2571         } else {
2572             ctb_addr_ts = hls_slice_data(s);
2573             if (ctb_addr_ts >= (s->ps.sps->ctb_width * s->ps.sps->ctb_height)) {
2574                 s->is_decoded = 1;
2575                 if ((s->ps.pps->transquant_bypass_enable_flag ||
2576                      (s->ps.sps->pcm.loop_filter_disable_flag && s->ps.sps->pcm_enabled_flag)) &&
2577                     s->ps.sps->sao_enabled)
2578                     restore_tqb_pixels(s);
2579             }
2580
2581             if (ctb_addr_ts < 0) {
2582                 ret = ctb_addr_ts;
2583                 goto fail;
2584             }
2585         }
2586         break;
2587     case HEVC_NAL_EOS_NUT:
2588     case HEVC_NAL_EOB_NUT:
2589         s->seq_decode = (s->seq_decode + 1) & 0xff;
2590         s->max_ra     = INT_MAX;
2591         break;
2592     case HEVC_NAL_AUD:
2593     case HEVC_NAL_FD_NUT:
2594         break;
2595     default:
2596         av_log(s->avctx, AV_LOG_INFO,
2597                "Skipping NAL unit %d\n", s->nal_unit_type);
2598     }
2599
2600     return 0;
2601 fail:
2602     if (s->avctx->err_recognition & AV_EF_EXPLODE)
2603         return ret;
2604     return 0;
2605 }
2606
2607 static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length)
2608 {
2609     int i, ret = 0;
2610
2611     s->ref = NULL;
2612     s->eos = 0;
2613
2614     /* split the input packet into NAL units, so we know the upper bound on the
2615      * number of slices in the frame */
2616     ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx, s->is_nalff,
2617                                 s->nal_length_size, s->avctx->codec_id);
2618     if (ret < 0) {
2619         av_log(s->avctx, AV_LOG_ERROR,
2620                "Error splitting the input into NAL units.\n");
2621         return ret;
2622     }
2623
2624     for (i = 0; i < s->pkt.nb_nals; i++) {
2625         if (s->pkt.nals[i].type == HEVC_NAL_EOB_NUT ||
2626             s->pkt.nals[i].type == HEVC_NAL_EOS_NUT)
2627             s->eos = 1;
2628     }
2629
2630     /* decode the NAL units */
2631     for (i = 0; i < s->pkt.nb_nals; i++) {
2632         ret = decode_nal_unit(s, &s->pkt.nals[i]);
2633         if (ret < 0) {
2634             av_log(s->avctx, AV_LOG_WARNING,
2635                    "Error parsing NAL unit #%d.\n", i);
2636             goto fail;
2637         }
2638     }
2639
2640 fail:
2641     if (s->ref)
2642         ff_thread_report_progress(&s->ref->tf, INT_MAX, 0);
2643
2644     return ret;
2645 }
2646
2647 static void print_md5(void *log_ctx, int level, uint8_t md5[16])
2648 {
2649     int i;
2650     for (i = 0; i < 16; i++)
2651         av_log(log_ctx, level, "%02"PRIx8, md5[i]);
2652 }
2653
2654 static int verify_md5(HEVCContext *s, AVFrame *frame)
2655 {
2656     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
2657     int pixel_shift;
2658     int i, j;
2659
2660     if (!desc)
2661         return AVERROR(EINVAL);
2662
2663     pixel_shift = desc->comp[0].depth > 8;
2664
2665     av_log(s->avctx, AV_LOG_DEBUG, "Verifying checksum for frame with POC %d: ",
2666            s->poc);
2667
2668     /* the checksums are LE, so we have to byteswap for >8bpp formats
2669      * on BE arches */
2670 #if HAVE_BIGENDIAN
2671     if (pixel_shift && !s->checksum_buf) {
2672         av_fast_malloc(&s->checksum_buf, &s->checksum_buf_size,
2673                        FFMAX3(frame->linesize[0], frame->linesize[1],
2674                               frame->linesize[2]));
2675         if (!s->checksum_buf)
2676             return AVERROR(ENOMEM);
2677     }
2678 #endif
2679
2680     for (i = 0; frame->data[i]; i++) {
2681         int width  = s->avctx->coded_width;
2682         int height = s->avctx->coded_height;
2683         int w = (i == 1 || i == 2) ? (width  >> desc->log2_chroma_w) : width;
2684         int h = (i == 1 || i == 2) ? (height >> desc->log2_chroma_h) : height;
2685         uint8_t md5[16];
2686
2687         av_md5_init(s->md5_ctx);
2688         for (j = 0; j < h; j++) {
2689             const uint8_t *src = frame->data[i] + j * frame->linesize[i];
2690 #if HAVE_BIGENDIAN
2691             if (pixel_shift) {
2692                 s->bdsp.bswap16_buf((uint16_t *) s->checksum_buf,
2693                                     (const uint16_t *) src, w);
2694                 src = s->checksum_buf;
2695             }
2696 #endif
2697             av_md5_update(s->md5_ctx, src, w << pixel_shift);
2698         }
2699         av_md5_final(s->md5_ctx, md5);
2700
2701         if (!memcmp(md5, s->sei.picture_hash.md5[i], 16)) {
2702             av_log   (s->avctx, AV_LOG_DEBUG, "plane %d - correct ", i);
2703             print_md5(s->avctx, AV_LOG_DEBUG, md5);
2704             av_log   (s->avctx, AV_LOG_DEBUG, "; ");
2705         } else {
2706             av_log   (s->avctx, AV_LOG_ERROR, "mismatching checksum of plane %d - ", i);
2707             print_md5(s->avctx, AV_LOG_ERROR, md5);
2708             av_log   (s->avctx, AV_LOG_ERROR, " != ");
2709             print_md5(s->avctx, AV_LOG_ERROR, s->sei.picture_hash.md5[i]);
2710             av_log   (s->avctx, AV_LOG_ERROR, "\n");
2711             return AVERROR_INVALIDDATA;
2712         }
2713     }
2714
2715     av_log(s->avctx, AV_LOG_DEBUG, "\n");
2716
2717     return 0;
2718 }
2719
2720 static int hevc_decode_extradata(HEVCContext *s, uint8_t *buf, int length)
2721 {
2722     AVCodecContext *avctx = s->avctx;
2723     GetByteContext gb;
2724     int ret, i;
2725
2726     bytestream2_init(&gb, buf, length);
2727
2728     if (length > 3 && (buf[0] || buf[1] || buf[2] > 1)) {
2729         /* It seems the extradata is encoded as hvcC format.
2730          * Temporarily, we support configurationVersion==0 until 14496-15 3rd
2731          * is finalized. When finalized, configurationVersion will be 1 and we
2732          * can recognize hvcC by checking if avctx->extradata[0]==1 or not. */
2733         int i, j, num_arrays, nal_len_size;
2734
2735         s->is_nalff = 1;
2736
2737         bytestream2_skip(&gb, 21);
2738         nal_len_size = (bytestream2_get_byte(&gb) & 3) + 1;
2739         num_arrays   = bytestream2_get_byte(&gb);
2740
2741         /* nal units in the hvcC always have length coded with 2 bytes,
2742          * so put a fake nal_length_size = 2 while parsing them */
2743         s->nal_length_size = 2;
2744
2745         /* Decode nal units from hvcC. */
2746         for (i = 0; i < num_arrays; i++) {
2747             int type = bytestream2_get_byte(&gb) & 0x3f;
2748             int cnt  = bytestream2_get_be16(&gb);
2749
2750             for (j = 0; j < cnt; j++) {
2751                 // +2 for the nal size field
2752                 int nalsize = bytestream2_peek_be16(&gb) + 2;
2753                 if (bytestream2_get_bytes_left(&gb) < nalsize) {
2754                     av_log(s->avctx, AV_LOG_ERROR,
2755                            "Invalid NAL unit size in extradata.\n");
2756                     return AVERROR_INVALIDDATA;
2757                 }
2758
2759                 ret = decode_nal_units(s, gb.buffer, nalsize);
2760                 if (ret < 0) {
2761                     av_log(avctx, AV_LOG_ERROR,
2762                            "Decoding nal unit %d %d from hvcC failed\n",
2763                            type, i);
2764                     return ret;
2765                 }
2766                 bytestream2_skip(&gb, nalsize);
2767             }
2768         }
2769
2770         /* Now store right nal length size, that will be used to parse
2771          * all other nals */
2772         s->nal_length_size = nal_len_size;
2773     } else {
2774         s->is_nalff = 0;
2775         ret = decode_nal_units(s, buf, length);
2776         if (ret < 0)
2777             return ret;
2778     }
2779
2780     /* export stream parameters from the first SPS */
2781     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
2782         if (s->ps.sps_list[i]) {
2783             const HEVCSPS *sps = (const HEVCSPS*)s->ps.sps_list[i]->data;
2784             export_stream_params(s->avctx, &s->ps, sps);
2785             break;
2786         }
2787     }
2788
2789     return 0;
2790 }
2791
2792 static int hevc_decode_frame(AVCodecContext *avctx, void *data, int *got_output,
2793                              AVPacket *avpkt)
2794 {
2795     int ret;
2796     int new_extradata_size;
2797     uint8_t *new_extradata;
2798     HEVCContext *s = avctx->priv_data;
2799
2800     if (!avpkt->size) {
2801         ret = ff_hevc_output_frame(s, data, 1);
2802         if (ret < 0)
2803             return ret;
2804
2805         *got_output = ret;
2806         return 0;
2807     }
2808
2809     new_extradata = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA,
2810                                             &new_extradata_size);
2811     if (new_extradata && new_extradata_size > 0) {
2812         ret = hevc_decode_extradata(s, new_extradata, new_extradata_size);
2813         if (ret < 0)
2814             return ret;
2815     }
2816
2817     s->ref = NULL;
2818     ret    = decode_nal_units(s, avpkt->data, avpkt->size);
2819     if (ret < 0)
2820         return ret;
2821
2822     if (avctx->hwaccel) {
2823         if (s->ref && avctx->hwaccel->end_frame(avctx) < 0)
2824             av_log(avctx, AV_LOG_ERROR,
2825                    "hardware accelerator failed to decode picture\n");
2826     } else {
2827         /* verify the SEI checksum */
2828         if (avctx->err_recognition & AV_EF_CRCCHECK && s->is_decoded &&
2829             s->sei.picture_hash.is_md5) {
2830             ret = verify_md5(s, s->ref->frame);
2831             if (ret < 0 && avctx->err_recognition & AV_EF_EXPLODE) {
2832                 ff_hevc_unref_frame(s, s->ref, ~0);
2833                 return ret;
2834             }
2835         }
2836     }
2837     s->sei.picture_hash.is_md5 = 0;
2838
2839     if (s->is_decoded) {
2840         av_log(avctx, AV_LOG_DEBUG, "Decoded frame with POC %d.\n", s->poc);
2841         s->is_decoded = 0;
2842     }
2843
2844     if (s->output_frame->buf[0]) {
2845         av_frame_move_ref(data, s->output_frame);
2846         *got_output = 1;
2847     }
2848
2849     return avpkt->size;
2850 }
2851
2852 static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
2853 {
2854     int ret = ff_thread_ref_frame(&dst->tf, &src->tf);
2855     if (ret < 0)
2856         return ret;
2857
2858     dst->tab_mvf_buf = av_buffer_ref(src->tab_mvf_buf);
2859     if (!dst->tab_mvf_buf)
2860         goto fail;
2861     dst->tab_mvf = src->tab_mvf;
2862
2863     dst->rpl_tab_buf = av_buffer_ref(src->rpl_tab_buf);
2864     if (!dst->rpl_tab_buf)
2865         goto fail;
2866     dst->rpl_tab = src->rpl_tab;
2867
2868     dst->rpl_buf = av_buffer_ref(src->rpl_buf);
2869     if (!dst->rpl_buf)
2870         goto fail;
2871
2872     dst->poc        = src->poc;
2873     dst->ctb_count  = src->ctb_count;
2874     dst->flags      = src->flags;
2875     dst->sequence   = src->sequence;
2876
2877     if (src->hwaccel_picture_private) {
2878         dst->hwaccel_priv_buf = av_buffer_ref(src->hwaccel_priv_buf);
2879         if (!dst->hwaccel_priv_buf)
2880             goto fail;
2881         dst->hwaccel_picture_private = dst->hwaccel_priv_buf->data;
2882     }
2883
2884     return 0;
2885 fail:
2886     ff_hevc_unref_frame(s, dst, ~0);
2887     return AVERROR(ENOMEM);
2888 }
2889
2890 static av_cold int hevc_decode_free(AVCodecContext *avctx)
2891 {
2892     HEVCContext       *s = avctx->priv_data;
2893     int i;
2894
2895     pic_arrays_free(s);
2896
2897     av_freep(&s->md5_ctx);
2898
2899     av_frame_free(&s->tmp_frame);
2900     av_frame_free(&s->output_frame);
2901
2902     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2903         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2904         av_frame_free(&s->DPB[i].frame);
2905     }
2906
2907     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++)
2908         av_buffer_unref(&s->ps.vps_list[i]);
2909     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++)
2910         av_buffer_unref(&s->ps.sps_list[i]);
2911     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++)
2912         av_buffer_unref(&s->ps.pps_list[i]);
2913
2914     ff_h2645_packet_uninit(&s->pkt);
2915
2916     return 0;
2917 }
2918
2919 static av_cold int hevc_init_context(AVCodecContext *avctx)
2920 {
2921     HEVCContext *s = avctx->priv_data;
2922     int i;
2923
2924     s->avctx = avctx;
2925
2926     s->tmp_frame = av_frame_alloc();
2927     if (!s->tmp_frame)
2928         goto fail;
2929
2930     s->output_frame = av_frame_alloc();
2931     if (!s->output_frame)
2932         goto fail;
2933
2934     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2935         s->DPB[i].frame = av_frame_alloc();
2936         if (!s->DPB[i].frame)
2937             goto fail;
2938         s->DPB[i].tf.f = s->DPB[i].frame;
2939     }
2940
2941     s->max_ra = INT_MAX;
2942
2943     s->md5_ctx = av_md5_alloc();
2944     if (!s->md5_ctx)
2945         goto fail;
2946
2947     ff_bswapdsp_init(&s->bdsp);
2948
2949     s->context_initialized = 1;
2950
2951     return 0;
2952
2953 fail:
2954     hevc_decode_free(avctx);
2955     return AVERROR(ENOMEM);
2956 }
2957
2958 static int hevc_update_thread_context(AVCodecContext *dst,
2959                                       const AVCodecContext *src)
2960 {
2961     HEVCContext *s  = dst->priv_data;
2962     HEVCContext *s0 = src->priv_data;
2963     int i, ret;
2964
2965     if (!s->context_initialized) {
2966         ret = hevc_init_context(dst);
2967         if (ret < 0)
2968             return ret;
2969     }
2970
2971     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2972         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2973         if (s0->DPB[i].frame->buf[0]) {
2974             ret = hevc_ref_frame(s, &s->DPB[i], &s0->DPB[i]);
2975             if (ret < 0)
2976                 return ret;
2977         }
2978     }
2979
2980     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++) {
2981         av_buffer_unref(&s->ps.vps_list[i]);
2982         if (s0->ps.vps_list[i]) {
2983             s->ps.vps_list[i] = av_buffer_ref(s0->ps.vps_list[i]);
2984             if (!s->ps.vps_list[i])
2985                 return AVERROR(ENOMEM);
2986         }
2987     }
2988
2989     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
2990         av_buffer_unref(&s->ps.sps_list[i]);
2991         if (s0->ps.sps_list[i]) {
2992             s->ps.sps_list[i] = av_buffer_ref(s0->ps.sps_list[i]);
2993             if (!s->ps.sps_list[i])
2994                 return AVERROR(ENOMEM);
2995         }
2996     }
2997
2998     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++) {
2999         av_buffer_unref(&s->ps.pps_list[i]);
3000         if (s0->ps.pps_list[i]) {
3001             s->ps.pps_list[i] = av_buffer_ref(s0->ps.pps_list[i]);
3002             if (!s->ps.pps_list[i])
3003                 return AVERROR(ENOMEM);
3004         }
3005     }
3006
3007     if (s->ps.sps != s0->ps.sps)
3008         ret = set_sps(s, s0->ps.sps, src->pix_fmt);
3009
3010     s->seq_decode = s0->seq_decode;
3011     s->seq_output = s0->seq_output;
3012     s->pocTid0    = s0->pocTid0;
3013     s->max_ra     = s0->max_ra;
3014
3015     s->is_nalff        = s0->is_nalff;
3016     s->nal_length_size = s0->nal_length_size;
3017
3018     if (s0->eos) {
3019         s->seq_decode = (s->seq_decode + 1) & 0xff;
3020         s->max_ra = INT_MAX;
3021     }
3022
3023     return 0;
3024 }
3025
3026 static av_cold int hevc_decode_init(AVCodecContext *avctx)
3027 {
3028     HEVCContext *s = avctx->priv_data;
3029     int ret;
3030
3031     avctx->internal->allocate_progress = 1;
3032
3033     ret = hevc_init_context(avctx);
3034     if (ret < 0)
3035         return ret;
3036
3037     if (avctx->extradata_size > 0 && avctx->extradata) {
3038         ret = hevc_decode_extradata(s, avctx->extradata, avctx->extradata_size);
3039         if (ret < 0) {
3040             hevc_decode_free(avctx);
3041             return ret;
3042         }
3043     }
3044
3045     return 0;
3046 }
3047
3048 static av_cold int hevc_init_thread_copy(AVCodecContext *avctx)
3049 {
3050     HEVCContext *s = avctx->priv_data;
3051     int ret;
3052
3053     memset(s, 0, sizeof(*s));
3054
3055     ret = hevc_init_context(avctx);
3056     if (ret < 0)
3057         return ret;
3058
3059     return 0;
3060 }
3061
3062 static void hevc_decode_flush(AVCodecContext *avctx)
3063 {
3064     HEVCContext *s = avctx->priv_data;
3065     ff_hevc_flush_dpb(s);
3066     s->max_ra = INT_MAX;
3067 }
3068
3069 #define OFFSET(x) offsetof(HEVCContext, x)
3070 #define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
3071
3072 static const AVOption options[] = {
3073     { "apply_defdispwin", "Apply default display window from VUI", OFFSET(apply_defdispwin),
3074         AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, PAR },
3075     { NULL },
3076 };
3077
3078 static const AVClass hevc_decoder_class = {
3079     .class_name = "HEVC decoder",
3080     .item_name  = av_default_item_name,
3081     .option     = options,
3082     .version    = LIBAVUTIL_VERSION_INT,
3083 };
3084
3085 AVCodec ff_hevc_decoder = {
3086     .name                  = "hevc",
3087     .long_name             = NULL_IF_CONFIG_SMALL("HEVC (High Efficiency Video Coding)"),
3088     .type                  = AVMEDIA_TYPE_VIDEO,
3089     .id                    = AV_CODEC_ID_HEVC,
3090     .priv_data_size        = sizeof(HEVCContext),
3091     .priv_class            = &hevc_decoder_class,
3092     .init                  = hevc_decode_init,
3093     .close                 = hevc_decode_free,
3094     .decode                = hevc_decode_frame,
3095     .flush                 = hevc_decode_flush,
3096     .update_thread_context = hevc_update_thread_context,
3097     .init_thread_copy      = hevc_init_thread_copy,
3098     .capabilities          = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY |
3099                              AV_CODEC_CAP_FRAME_THREADS,
3100     .profiles              = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
3101     .caps_internal         = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_THREADSAFE,
3102 };