hevcdec: add a CUVID hwaccel
[ffmpeg.git] / libavcodec / hevcdec.c
1 /*
2  * HEVC video decoder
3  *
4  * Copyright (C) 2012 - 2013 Guillaume Martres
5  * Copyright (C) 2012 - 2013 Mickael Raulet
6  * Copyright (C) 2012 - 2013 Gildas Cocherel
7  * Copyright (C) 2012 - 2013 Wassim Hamidouche
8  *
9  * This file is part of Libav.
10  *
11  * Libav is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU Lesser General Public
13  * License as published by the Free Software Foundation; either
14  * version 2.1 of the License, or (at your option) any later version.
15  *
16  * Libav is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19  * Lesser General Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser General Public
22  * License along with Libav; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24  */
25
26 #include "libavutil/attributes.h"
27 #include "libavutil/common.h"
28 #include "libavutil/display.h"
29 #include "libavutil/internal.h"
30 #include "libavutil/md5.h"
31 #include "libavutil/opt.h"
32 #include "libavutil/pixdesc.h"
33 #include "libavutil/stereo3d.h"
34
35 #include "bswapdsp.h"
36 #include "bytestream.h"
37 #include "cabac_functions.h"
38 #include "golomb_legacy.h"
39 #include "hevc.h"
40 #include "hevc_data.h"
41 #include "hevcdec.h"
42 #include "profiles.h"
43
44 const uint8_t ff_hevc_qpel_extra_before[4] = { 0, 3, 3, 3 };
45 const uint8_t ff_hevc_qpel_extra_after[4]  = { 0, 4, 4, 4 };
46 const uint8_t ff_hevc_qpel_extra[4]        = { 0, 7, 7, 7 };
47
48 static const uint8_t scan_1x1[1] = { 0 };
49
50 static const uint8_t horiz_scan2x2_x[4] = { 0, 1, 0, 1 };
51
52 static const uint8_t horiz_scan2x2_y[4] = { 0, 0, 1, 1 };
53
54 static const uint8_t horiz_scan4x4_x[16] = {
55     0, 1, 2, 3,
56     0, 1, 2, 3,
57     0, 1, 2, 3,
58     0, 1, 2, 3,
59 };
60
61 static const uint8_t horiz_scan4x4_y[16] = {
62     0, 0, 0, 0,
63     1, 1, 1, 1,
64     2, 2, 2, 2,
65     3, 3, 3, 3,
66 };
67
68 static const uint8_t horiz_scan8x8_inv[8][8] = {
69     {  0,  1,  2,  3, 16, 17, 18, 19, },
70     {  4,  5,  6,  7, 20, 21, 22, 23, },
71     {  8,  9, 10, 11, 24, 25, 26, 27, },
72     { 12, 13, 14, 15, 28, 29, 30, 31, },
73     { 32, 33, 34, 35, 48, 49, 50, 51, },
74     { 36, 37, 38, 39, 52, 53, 54, 55, },
75     { 40, 41, 42, 43, 56, 57, 58, 59, },
76     { 44, 45, 46, 47, 60, 61, 62, 63, },
77 };
78
79 static const uint8_t diag_scan2x2_x[4] = { 0, 0, 1, 1 };
80
81 static const uint8_t diag_scan2x2_y[4] = { 0, 1, 0, 1 };
82
83 static const uint8_t diag_scan2x2_inv[2][2] = {
84     { 0, 2, },
85     { 1, 3, },
86 };
87
88 static const uint8_t diag_scan4x4_inv[4][4] = {
89     { 0,  2,  5,  9, },
90     { 1,  4,  8, 12, },
91     { 3,  7, 11, 14, },
92     { 6, 10, 13, 15, },
93 };
94
95 static const uint8_t diag_scan8x8_inv[8][8] = {
96     {  0,  2,  5,  9, 14, 20, 27, 35, },
97     {  1,  4,  8, 13, 19, 26, 34, 42, },
98     {  3,  7, 12, 18, 25, 33, 41, 48, },
99     {  6, 11, 17, 24, 32, 40, 47, 53, },
100     { 10, 16, 23, 31, 39, 46, 52, 57, },
101     { 15, 22, 30, 38, 45, 51, 56, 60, },
102     { 21, 29, 37, 44, 50, 55, 59, 62, },
103     { 28, 36, 43, 49, 54, 58, 61, 63, },
104 };
105
106 /**
107  * NOTE: Each function hls_foo correspond to the function foo in the
108  * specification (HLS stands for High Level Syntax).
109  */
110
111 /**
112  * Section 5.7
113  */
114
115 /* free everything allocated  by pic_arrays_init() */
116 static void pic_arrays_free(HEVCContext *s)
117 {
118     av_freep(&s->sao);
119     av_freep(&s->deblock);
120
121     av_freep(&s->skip_flag);
122     av_freep(&s->tab_ct_depth);
123
124     av_freep(&s->tab_ipm);
125     av_freep(&s->cbf_luma);
126     av_freep(&s->is_pcm);
127
128     av_freep(&s->qp_y_tab);
129     av_freep(&s->tab_slice_address);
130     av_freep(&s->filter_slice_edges);
131
132     av_freep(&s->horizontal_bs);
133     av_freep(&s->vertical_bs);
134
135     av_buffer_pool_uninit(&s->tab_mvf_pool);
136     av_buffer_pool_uninit(&s->rpl_tab_pool);
137 }
138
139 /* allocate arrays that depend on frame dimensions */
140 static int pic_arrays_init(HEVCContext *s, const HEVCSPS *sps)
141 {
142     int log2_min_cb_size = sps->log2_min_cb_size;
143     int width            = sps->width;
144     int height           = sps->height;
145     int pic_size_in_ctb  = ((width  >> log2_min_cb_size) + 1) *
146                            ((height >> log2_min_cb_size) + 1);
147     int ctb_count        = sps->ctb_width * sps->ctb_height;
148     int min_pu_size      = sps->min_pu_width * sps->min_pu_height;
149
150     s->bs_width  = width  >> 3;
151     s->bs_height = height >> 3;
152
153     s->sao           = av_mallocz_array(ctb_count, sizeof(*s->sao));
154     s->deblock       = av_mallocz_array(ctb_count, sizeof(*s->deblock));
155     if (!s->sao || !s->deblock)
156         goto fail;
157
158     s->skip_flag    = av_malloc(pic_size_in_ctb);
159     s->tab_ct_depth = av_malloc(sps->min_cb_height * sps->min_cb_width);
160     if (!s->skip_flag || !s->tab_ct_depth)
161         goto fail;
162
163     s->cbf_luma = av_malloc(sps->min_tb_width * sps->min_tb_height);
164     s->tab_ipm  = av_mallocz(min_pu_size);
165     s->is_pcm   = av_malloc(min_pu_size);
166     if (!s->tab_ipm || !s->cbf_luma || !s->is_pcm)
167         goto fail;
168
169     s->filter_slice_edges = av_malloc(ctb_count);
170     s->tab_slice_address  = av_malloc(pic_size_in_ctb *
171                                       sizeof(*s->tab_slice_address));
172     s->qp_y_tab           = av_malloc(pic_size_in_ctb *
173                                       sizeof(*s->qp_y_tab));
174     if (!s->qp_y_tab || !s->filter_slice_edges || !s->tab_slice_address)
175         goto fail;
176
177     s->horizontal_bs = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
178     s->vertical_bs   = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
179     if (!s->horizontal_bs || !s->vertical_bs)
180         goto fail;
181
182     s->tab_mvf_pool = av_buffer_pool_init(min_pu_size * sizeof(MvField),
183                                           av_buffer_alloc);
184     s->rpl_tab_pool = av_buffer_pool_init(ctb_count * sizeof(RefPicListTab),
185                                           av_buffer_allocz);
186     if (!s->tab_mvf_pool || !s->rpl_tab_pool)
187         goto fail;
188
189     return 0;
190
191 fail:
192     pic_arrays_free(s);
193     return AVERROR(ENOMEM);
194 }
195
196 static void pred_weight_table(HEVCContext *s, GetBitContext *gb)
197 {
198     int i = 0;
199     int j = 0;
200     uint8_t luma_weight_l0_flag[16];
201     uint8_t chroma_weight_l0_flag[16];
202     uint8_t luma_weight_l1_flag[16];
203     uint8_t chroma_weight_l1_flag[16];
204
205     s->sh.luma_log2_weight_denom = av_clip(get_ue_golomb_long(gb), 0, 7);
206     if (s->ps.sps->chroma_format_idc != 0) {
207         int delta = get_se_golomb(gb);
208         s->sh.chroma_log2_weight_denom = av_clip(s->sh.luma_log2_weight_denom + delta, 0, 7);
209     }
210
211     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
212         luma_weight_l0_flag[i] = get_bits1(gb);
213         if (!luma_weight_l0_flag[i]) {
214             s->sh.luma_weight_l0[i] = 1 << s->sh.luma_log2_weight_denom;
215             s->sh.luma_offset_l0[i] = 0;
216         }
217     }
218     if (s->ps.sps->chroma_format_idc != 0) { // FIXME: invert "if" and "for"
219         for (i = 0; i < s->sh.nb_refs[L0]; i++)
220             chroma_weight_l0_flag[i] = get_bits1(gb);
221     } else {
222         for (i = 0; i < s->sh.nb_refs[L0]; i++)
223             chroma_weight_l0_flag[i] = 0;
224     }
225     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
226         if (luma_weight_l0_flag[i]) {
227             int delta_luma_weight_l0 = get_se_golomb(gb);
228             s->sh.luma_weight_l0[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l0;
229             s->sh.luma_offset_l0[i] = get_se_golomb(gb);
230         }
231         if (chroma_weight_l0_flag[i]) {
232             for (j = 0; j < 2; j++) {
233                 int delta_chroma_weight_l0 = get_se_golomb(gb);
234                 int delta_chroma_offset_l0 = get_se_golomb(gb);
235                 s->sh.chroma_weight_l0[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l0;
236                 s->sh.chroma_offset_l0[i][j] = av_clip((delta_chroma_offset_l0 - ((128 * s->sh.chroma_weight_l0[i][j])
237                                                                                     >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
238             }
239         } else {
240             s->sh.chroma_weight_l0[i][0] = 1 << s->sh.chroma_log2_weight_denom;
241             s->sh.chroma_offset_l0[i][0] = 0;
242             s->sh.chroma_weight_l0[i][1] = 1 << s->sh.chroma_log2_weight_denom;
243             s->sh.chroma_offset_l0[i][1] = 0;
244         }
245     }
246     if (s->sh.slice_type == HEVC_SLICE_B) {
247         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
248             luma_weight_l1_flag[i] = get_bits1(gb);
249             if (!luma_weight_l1_flag[i]) {
250                 s->sh.luma_weight_l1[i] = 1 << s->sh.luma_log2_weight_denom;
251                 s->sh.luma_offset_l1[i] = 0;
252             }
253         }
254         if (s->ps.sps->chroma_format_idc != 0) {
255             for (i = 0; i < s->sh.nb_refs[L1]; i++)
256                 chroma_weight_l1_flag[i] = get_bits1(gb);
257         } else {
258             for (i = 0; i < s->sh.nb_refs[L1]; i++)
259                 chroma_weight_l1_flag[i] = 0;
260         }
261         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
262             if (luma_weight_l1_flag[i]) {
263                 int delta_luma_weight_l1 = get_se_golomb(gb);
264                 s->sh.luma_weight_l1[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l1;
265                 s->sh.luma_offset_l1[i] = get_se_golomb(gb);
266             }
267             if (chroma_weight_l1_flag[i]) {
268                 for (j = 0; j < 2; j++) {
269                     int delta_chroma_weight_l1 = get_se_golomb(gb);
270                     int delta_chroma_offset_l1 = get_se_golomb(gb);
271                     s->sh.chroma_weight_l1[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l1;
272                     s->sh.chroma_offset_l1[i][j] = av_clip((delta_chroma_offset_l1 - ((128 * s->sh.chroma_weight_l1[i][j])
273                                                                                         >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
274                 }
275             } else {
276                 s->sh.chroma_weight_l1[i][0] = 1 << s->sh.chroma_log2_weight_denom;
277                 s->sh.chroma_offset_l1[i][0] = 0;
278                 s->sh.chroma_weight_l1[i][1] = 1 << s->sh.chroma_log2_weight_denom;
279                 s->sh.chroma_offset_l1[i][1] = 0;
280             }
281         }
282     }
283 }
284
285 static int decode_lt_rps(HEVCContext *s, LongTermRPS *rps, GetBitContext *gb)
286 {
287     const HEVCSPS *sps = s->ps.sps;
288     int max_poc_lsb    = 1 << sps->log2_max_poc_lsb;
289     int prev_delta_msb = 0;
290     unsigned int nb_sps = 0, nb_sh;
291     int i;
292
293     rps->nb_refs = 0;
294     if (!sps->long_term_ref_pics_present_flag)
295         return 0;
296
297     if (sps->num_long_term_ref_pics_sps > 0)
298         nb_sps = get_ue_golomb_long(gb);
299     nb_sh = get_ue_golomb_long(gb);
300
301     if (nb_sh + nb_sps > FF_ARRAY_ELEMS(rps->poc))
302         return AVERROR_INVALIDDATA;
303
304     rps->nb_refs = nb_sh + nb_sps;
305
306     for (i = 0; i < rps->nb_refs; i++) {
307         uint8_t delta_poc_msb_present;
308
309         if (i < nb_sps) {
310             uint8_t lt_idx_sps = 0;
311
312             if (sps->num_long_term_ref_pics_sps > 1)
313                 lt_idx_sps = get_bits(gb, av_ceil_log2(sps->num_long_term_ref_pics_sps));
314
315             rps->poc[i]  = sps->lt_ref_pic_poc_lsb_sps[lt_idx_sps];
316             rps->used[i] = sps->used_by_curr_pic_lt_sps_flag[lt_idx_sps];
317         } else {
318             rps->poc[i]  = get_bits(gb, sps->log2_max_poc_lsb);
319             rps->used[i] = get_bits1(gb);
320         }
321
322         delta_poc_msb_present = get_bits1(gb);
323         if (delta_poc_msb_present) {
324             int delta = get_ue_golomb_long(gb);
325
326             if (i && i != nb_sps)
327                 delta += prev_delta_msb;
328
329             rps->poc[i] += s->poc - delta * max_poc_lsb - s->sh.pic_order_cnt_lsb;
330             prev_delta_msb = delta;
331         }
332     }
333
334     return 0;
335 }
336
337 static void export_stream_params(AVCodecContext *avctx, const HEVCParamSets *ps,
338                                  const HEVCSPS *sps)
339 {
340     const HEVCVPS *vps = (const HEVCVPS*)ps->vps_list[sps->vps_id]->data;
341     const HEVCWindow *ow = &sps->output_window;
342     unsigned int num = 0, den = 0;
343
344     avctx->pix_fmt             = sps->pix_fmt;
345     avctx->coded_width         = sps->width;
346     avctx->coded_height        = sps->height;
347     avctx->width               = sps->width  - ow->left_offset - ow->right_offset;
348     avctx->height              = sps->height - ow->top_offset  - ow->bottom_offset;
349     avctx->has_b_frames        = sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics;
350     avctx->profile             = sps->ptl.general_ptl.profile_idc;
351     avctx->level               = sps->ptl.general_ptl.level_idc;
352
353     ff_set_sar(avctx, sps->vui.sar);
354
355     if (sps->vui.video_signal_type_present_flag)
356         avctx->color_range = sps->vui.video_full_range_flag ? AVCOL_RANGE_JPEG
357                                                             : AVCOL_RANGE_MPEG;
358     else
359         avctx->color_range = AVCOL_RANGE_MPEG;
360
361     if (sps->vui.colour_description_present_flag) {
362         avctx->color_primaries = sps->vui.colour_primaries;
363         avctx->color_trc       = sps->vui.transfer_characteristic;
364         avctx->colorspace      = sps->vui.matrix_coeffs;
365     } else {
366         avctx->color_primaries = AVCOL_PRI_UNSPECIFIED;
367         avctx->color_trc       = AVCOL_TRC_UNSPECIFIED;
368         avctx->colorspace      = AVCOL_SPC_UNSPECIFIED;
369     }
370
371     if (vps->vps_timing_info_present_flag) {
372         num = vps->vps_num_units_in_tick;
373         den = vps->vps_time_scale;
374     } else if (sps->vui.vui_timing_info_present_flag) {
375         num = sps->vui.vui_num_units_in_tick;
376         den = sps->vui.vui_time_scale;
377     }
378
379     if (num != 0 && den != 0)
380         av_reduce(&avctx->framerate.den, &avctx->framerate.num,
381                   num, den, 1 << 30);
382 }
383
384 static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
385 {
386     #define HWACCEL_MAX (CONFIG_HEVC_DXVA2_HWACCEL + CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \
387                          CONFIG_HEVC_VAAPI_HWACCEL + CONFIG_HEVC_VDPAU_HWACCEL + \
388                          CONFIG_HEVC_CUVID_HWACCEL)
389     enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmt = pix_fmts;
390
391     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P ||
392         sps->pix_fmt == AV_PIX_FMT_YUV420P10) {
393 #if CONFIG_HEVC_D3D11VA_HWACCEL
394         *fmt++ = AV_PIX_FMT_D3D11VA_VLD;
395         *fmt++ = AV_PIX_FMT_D3D11;
396 #endif
397 #if CONFIG_HEVC_DXVA2_HWACCEL
398         *fmt++ = AV_PIX_FMT_DXVA2_VLD;
399 #endif
400 #if CONFIG_HEVC_VAAPI_HWACCEL
401         *fmt++ = AV_PIX_FMT_VAAPI;
402 #endif
403 #if CONFIG_HEVC_CUVID_HWACCEL && HAVE_CUVIDDECODECREATEINFO_BITDEPTHMINUS8
404         *fmt++ = AV_PIX_FMT_CUDA;
405 #endif
406     }
407     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P) {
408 #if CONFIG_HEVC_CUVID_HWACCEL && !HAVE_CUVIDDECODECREATEINFO_BITDEPTHMINUS8
409         *fmt++ = AV_PIX_FMT_CUDA;
410 #endif
411 #if CONFIG_HEVC_VDPAU_HWACCEL
412         *fmt++ = AV_PIX_FMT_VDPAU;
413 #endif
414     }
415
416     *fmt++ = sps->pix_fmt;
417     *fmt = AV_PIX_FMT_NONE;
418
419     return ff_get_format(s->avctx, pix_fmts);
420 }
421
422 static int set_sps(HEVCContext *s, const HEVCSPS *sps,
423                    enum AVPixelFormat pix_fmt)
424 {
425     int ret;
426
427     pic_arrays_free(s);
428     s->ps.sps = NULL;
429     s->ps.vps = NULL;
430
431     if (!sps)
432         return 0;
433
434     ret = pic_arrays_init(s, sps);
435     if (ret < 0)
436         goto fail;
437
438     export_stream_params(s->avctx, &s->ps, sps);
439
440     s->avctx->pix_fmt = pix_fmt;
441
442     ff_hevc_pred_init(&s->hpc,     sps->bit_depth);
443     ff_hevc_dsp_init (&s->hevcdsp, sps->bit_depth);
444     ff_videodsp_init (&s->vdsp,    sps->bit_depth);
445
446     if (sps->sao_enabled && !s->avctx->hwaccel) {
447         av_frame_unref(s->tmp_frame);
448         ret = ff_get_buffer(s->avctx, s->tmp_frame, AV_GET_BUFFER_FLAG_REF);
449         if (ret < 0)
450             goto fail;
451         s->frame = s->tmp_frame;
452     }
453
454     s->ps.sps = sps;
455     s->ps.vps = (HEVCVPS*) s->ps.vps_list[s->ps.sps->vps_id]->data;
456
457     return 0;
458
459 fail:
460     pic_arrays_free(s);
461     s->ps.sps = NULL;
462     return ret;
463 }
464
465 static int hls_slice_header(HEVCContext *s)
466 {
467     GetBitContext *gb = &s->HEVClc.gb;
468     SliceHeader *sh   = &s->sh;
469     int i, ret;
470
471     // Coded parameters
472     sh->first_slice_in_pic_flag = get_bits1(gb);
473     if ((IS_IDR(s) || IS_BLA(s)) && sh->first_slice_in_pic_flag) {
474         s->seq_decode = (s->seq_decode + 1) & 0xff;
475         s->max_ra     = INT_MAX;
476         if (IS_IDR(s))
477             ff_hevc_clear_refs(s);
478     }
479     if (IS_IRAP(s))
480         sh->no_output_of_prior_pics_flag = get_bits1(gb);
481
482     sh->pps_id = get_ue_golomb_long(gb);
483     if (sh->pps_id >= HEVC_MAX_PPS_COUNT || !s->ps.pps_list[sh->pps_id]) {
484         av_log(s->avctx, AV_LOG_ERROR, "PPS id out of range: %d\n", sh->pps_id);
485         return AVERROR_INVALIDDATA;
486     }
487     if (!sh->first_slice_in_pic_flag &&
488         s->ps.pps != (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data) {
489         av_log(s->avctx, AV_LOG_ERROR, "PPS changed between slices.\n");
490         return AVERROR_INVALIDDATA;
491     }
492     s->ps.pps = (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data;
493
494     if (s->ps.sps != (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data) {
495         const HEVCSPS *sps = (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data;
496         enum AVPixelFormat pix_fmt;
497
498         ff_hevc_clear_refs(s);
499
500         ret = set_sps(s, sps, sps->pix_fmt);
501         if (ret < 0)
502             return ret;
503
504         pix_fmt = get_format(s, sps);
505         if (pix_fmt < 0)
506             return pix_fmt;
507         s->avctx->pix_fmt = pix_fmt;
508
509         s->seq_decode = (s->seq_decode + 1) & 0xff;
510         s->max_ra     = INT_MAX;
511     }
512
513     sh->dependent_slice_segment_flag = 0;
514     if (!sh->first_slice_in_pic_flag) {
515         int slice_address_length;
516
517         if (s->ps.pps->dependent_slice_segments_enabled_flag)
518             sh->dependent_slice_segment_flag = get_bits1(gb);
519
520         slice_address_length = av_ceil_log2(s->ps.sps->ctb_width *
521                                             s->ps.sps->ctb_height);
522         sh->slice_segment_addr = slice_address_length ? get_bits(gb, slice_address_length) : 0;
523         if (sh->slice_segment_addr >= s->ps.sps->ctb_width * s->ps.sps->ctb_height) {
524             av_log(s->avctx, AV_LOG_ERROR,
525                    "Invalid slice segment address: %u.\n",
526                    sh->slice_segment_addr);
527             return AVERROR_INVALIDDATA;
528         }
529
530         if (!sh->dependent_slice_segment_flag) {
531             sh->slice_addr = sh->slice_segment_addr;
532             s->slice_idx++;
533         }
534     } else {
535         sh->slice_segment_addr = sh->slice_addr = 0;
536         s->slice_idx           = 0;
537         s->slice_initialized   = 0;
538     }
539
540     if (!sh->dependent_slice_segment_flag) {
541         s->slice_initialized = 0;
542
543         for (i = 0; i < s->ps.pps->num_extra_slice_header_bits; i++)
544             skip_bits(gb, 1);  // slice_reserved_undetermined_flag[]
545
546         sh->slice_type = get_ue_golomb_long(gb);
547         if (!(sh->slice_type == HEVC_SLICE_I ||
548               sh->slice_type == HEVC_SLICE_P ||
549               sh->slice_type == HEVC_SLICE_B)) {
550             av_log(s->avctx, AV_LOG_ERROR, "Unknown slice type: %d.\n",
551                    sh->slice_type);
552             return AVERROR_INVALIDDATA;
553         }
554         if (IS_IRAP(s) && sh->slice_type != HEVC_SLICE_I) {
555             av_log(s->avctx, AV_LOG_ERROR, "Inter slices in an IRAP frame.\n");
556             return AVERROR_INVALIDDATA;
557         }
558
559         // when flag is not present, picture is inferred to be output
560         sh->pic_output_flag = 1;
561         if (s->ps.pps->output_flag_present_flag)
562             sh->pic_output_flag = get_bits1(gb);
563
564         if (s->ps.sps->separate_colour_plane_flag)
565             sh->colour_plane_id = get_bits(gb, 2);
566
567         if (!IS_IDR(s)) {
568             int poc, pos;
569
570             sh->pic_order_cnt_lsb = get_bits(gb, s->ps.sps->log2_max_poc_lsb);
571             poc = ff_hevc_compute_poc(s, sh->pic_order_cnt_lsb);
572             if (!sh->first_slice_in_pic_flag && poc != s->poc) {
573                 av_log(s->avctx, AV_LOG_WARNING,
574                        "Ignoring POC change between slices: %d -> %d\n", s->poc, poc);
575                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
576                     return AVERROR_INVALIDDATA;
577                 poc = s->poc;
578             }
579             s->poc = poc;
580
581             sh->short_term_ref_pic_set_sps_flag = get_bits1(gb);
582             pos = get_bits_left(gb);
583             if (!sh->short_term_ref_pic_set_sps_flag) {
584                 ret = ff_hevc_decode_short_term_rps(gb, s->avctx, &sh->slice_rps, s->ps.sps, 1);
585                 if (ret < 0)
586                     return ret;
587
588                 sh->short_term_rps = &sh->slice_rps;
589             } else {
590                 int numbits, rps_idx;
591
592                 if (!s->ps.sps->nb_st_rps) {
593                     av_log(s->avctx, AV_LOG_ERROR, "No ref lists in the SPS.\n");
594                     return AVERROR_INVALIDDATA;
595                 }
596
597                 numbits = av_ceil_log2(s->ps.sps->nb_st_rps);
598                 rps_idx = numbits > 0 ? get_bits(gb, numbits) : 0;
599                 sh->short_term_rps = &s->ps.sps->st_rps[rps_idx];
600             }
601             sh->short_term_ref_pic_set_size = pos - get_bits_left(gb);
602
603             pos = get_bits_left(gb);
604             ret = decode_lt_rps(s, &sh->long_term_rps, gb);
605             if (ret < 0) {
606                 av_log(s->avctx, AV_LOG_WARNING, "Invalid long term RPS.\n");
607                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
608                     return AVERROR_INVALIDDATA;
609             }
610             sh->long_term_ref_pic_set_size = pos - get_bits_left(gb);
611
612             if (s->ps.sps->sps_temporal_mvp_enabled_flag)
613                 sh->slice_temporal_mvp_enabled_flag = get_bits1(gb);
614             else
615                 sh->slice_temporal_mvp_enabled_flag = 0;
616         } else {
617             s->sh.short_term_rps = NULL;
618             s->poc               = 0;
619         }
620
621         /* 8.3.1 */
622         if (s->temporal_id == 0 &&
623             s->nal_unit_type != HEVC_NAL_TRAIL_N &&
624             s->nal_unit_type != HEVC_NAL_TSA_N   &&
625             s->nal_unit_type != HEVC_NAL_STSA_N  &&
626             s->nal_unit_type != HEVC_NAL_RADL_N  &&
627             s->nal_unit_type != HEVC_NAL_RADL_R  &&
628             s->nal_unit_type != HEVC_NAL_RASL_N  &&
629             s->nal_unit_type != HEVC_NAL_RASL_R)
630             s->pocTid0 = s->poc;
631
632         if (s->ps.sps->sao_enabled) {
633             sh->slice_sample_adaptive_offset_flag[0] = get_bits1(gb);
634             sh->slice_sample_adaptive_offset_flag[1] =
635             sh->slice_sample_adaptive_offset_flag[2] = get_bits1(gb);
636         } else {
637             sh->slice_sample_adaptive_offset_flag[0] = 0;
638             sh->slice_sample_adaptive_offset_flag[1] = 0;
639             sh->slice_sample_adaptive_offset_flag[2] = 0;
640         }
641
642         sh->nb_refs[L0] = sh->nb_refs[L1] = 0;
643         if (sh->slice_type == HEVC_SLICE_P || sh->slice_type == HEVC_SLICE_B) {
644             int nb_refs;
645
646             sh->nb_refs[L0] = s->ps.pps->num_ref_idx_l0_default_active;
647             if (sh->slice_type == HEVC_SLICE_B)
648                 sh->nb_refs[L1] = s->ps.pps->num_ref_idx_l1_default_active;
649
650             if (get_bits1(gb)) { // num_ref_idx_active_override_flag
651                 sh->nb_refs[L0] = get_ue_golomb_long(gb) + 1;
652                 if (sh->slice_type == HEVC_SLICE_B)
653                     sh->nb_refs[L1] = get_ue_golomb_long(gb) + 1;
654             }
655             if (sh->nb_refs[L0] > HEVC_MAX_REFS || sh->nb_refs[L1] > HEVC_MAX_REFS) {
656                 av_log(s->avctx, AV_LOG_ERROR, "Too many refs: %d/%d.\n",
657                        sh->nb_refs[L0], sh->nb_refs[L1]);
658                 return AVERROR_INVALIDDATA;
659             }
660
661             sh->rpl_modification_flag[0] = 0;
662             sh->rpl_modification_flag[1] = 0;
663             nb_refs = ff_hevc_frame_nb_refs(s);
664             if (!nb_refs) {
665                 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
666                 return AVERROR_INVALIDDATA;
667             }
668
669             if (s->ps.pps->lists_modification_present_flag && nb_refs > 1) {
670                 sh->rpl_modification_flag[0] = get_bits1(gb);
671                 if (sh->rpl_modification_flag[0]) {
672                     for (i = 0; i < sh->nb_refs[L0]; i++)
673                         sh->list_entry_lx[0][i] = get_bits(gb, av_ceil_log2(nb_refs));
674                 }
675
676                 if (sh->slice_type == HEVC_SLICE_B) {
677                     sh->rpl_modification_flag[1] = get_bits1(gb);
678                     if (sh->rpl_modification_flag[1] == 1)
679                         for (i = 0; i < sh->nb_refs[L1]; i++)
680                             sh->list_entry_lx[1][i] = get_bits(gb, av_ceil_log2(nb_refs));
681                 }
682             }
683
684             if (sh->slice_type == HEVC_SLICE_B)
685                 sh->mvd_l1_zero_flag = get_bits1(gb);
686
687             if (s->ps.pps->cabac_init_present_flag)
688                 sh->cabac_init_flag = get_bits1(gb);
689             else
690                 sh->cabac_init_flag = 0;
691
692             sh->collocated_ref_idx = 0;
693             if (sh->slice_temporal_mvp_enabled_flag) {
694                 sh->collocated_list = L0;
695                 if (sh->slice_type == HEVC_SLICE_B)
696                     sh->collocated_list = !get_bits1(gb);
697
698                 if (sh->nb_refs[sh->collocated_list] > 1) {
699                     sh->collocated_ref_idx = get_ue_golomb_long(gb);
700                     if (sh->collocated_ref_idx >= sh->nb_refs[sh->collocated_list]) {
701                         av_log(s->avctx, AV_LOG_ERROR,
702                                "Invalid collocated_ref_idx: %d.\n",
703                                sh->collocated_ref_idx);
704                         return AVERROR_INVALIDDATA;
705                     }
706                 }
707             }
708
709             if ((s->ps.pps->weighted_pred_flag   && sh->slice_type == HEVC_SLICE_P) ||
710                 (s->ps.pps->weighted_bipred_flag && sh->slice_type == HEVC_SLICE_B)) {
711                 pred_weight_table(s, gb);
712             }
713
714             sh->max_num_merge_cand = 5 - get_ue_golomb_long(gb);
715             if (sh->max_num_merge_cand < 1 || sh->max_num_merge_cand > 5) {
716                 av_log(s->avctx, AV_LOG_ERROR,
717                        "Invalid number of merging MVP candidates: %d.\n",
718                        sh->max_num_merge_cand);
719                 return AVERROR_INVALIDDATA;
720             }
721         }
722
723         sh->slice_qp_delta = get_se_golomb(gb);
724
725         if (s->ps.pps->pic_slice_level_chroma_qp_offsets_present_flag) {
726             sh->slice_cb_qp_offset = get_se_golomb(gb);
727             sh->slice_cr_qp_offset = get_se_golomb(gb);
728         } else {
729             sh->slice_cb_qp_offset = 0;
730             sh->slice_cr_qp_offset = 0;
731         }
732
733         if (s->ps.pps->deblocking_filter_control_present_flag) {
734             int deblocking_filter_override_flag = 0;
735
736             if (s->ps.pps->deblocking_filter_override_enabled_flag)
737                 deblocking_filter_override_flag = get_bits1(gb);
738
739             if (deblocking_filter_override_flag) {
740                 sh->disable_deblocking_filter_flag = get_bits1(gb);
741                 if (!sh->disable_deblocking_filter_flag) {
742                     sh->beta_offset = get_se_golomb(gb) * 2;
743                     sh->tc_offset   = get_se_golomb(gb) * 2;
744                 }
745             } else {
746                 sh->disable_deblocking_filter_flag = s->ps.pps->disable_dbf;
747                 sh->beta_offset                    = s->ps.pps->beta_offset;
748                 sh->tc_offset                      = s->ps.pps->tc_offset;
749             }
750         } else {
751             sh->disable_deblocking_filter_flag = 0;
752             sh->beta_offset                    = 0;
753             sh->tc_offset                      = 0;
754         }
755
756         if (s->ps.pps->seq_loop_filter_across_slices_enabled_flag &&
757             (sh->slice_sample_adaptive_offset_flag[0] ||
758              sh->slice_sample_adaptive_offset_flag[1] ||
759              !sh->disable_deblocking_filter_flag)) {
760             sh->slice_loop_filter_across_slices_enabled_flag = get_bits1(gb);
761         } else {
762             sh->slice_loop_filter_across_slices_enabled_flag = s->ps.pps->seq_loop_filter_across_slices_enabled_flag;
763         }
764     } else if (!s->slice_initialized) {
765         av_log(s->avctx, AV_LOG_ERROR, "Independent slice segment missing.\n");
766         return AVERROR_INVALIDDATA;
767     }
768
769     sh->num_entry_point_offsets = 0;
770     if (s->ps.pps->tiles_enabled_flag || s->ps.pps->entropy_coding_sync_enabled_flag) {
771         sh->num_entry_point_offsets = get_ue_golomb_long(gb);
772         if (sh->num_entry_point_offsets > 0) {
773             int offset_len = get_ue_golomb_long(gb) + 1;
774
775             for (i = 0; i < sh->num_entry_point_offsets; i++)
776                 skip_bits(gb, offset_len);
777         }
778     }
779
780     if (s->ps.pps->slice_header_extension_present_flag) {
781         unsigned int length = get_ue_golomb_long(gb);
782         for (i = 0; i < length; i++)
783             skip_bits(gb, 8);  // slice_header_extension_data_byte
784     }
785
786     // Inferred parameters
787     sh->slice_qp = 26 + s->ps.pps->pic_init_qp_minus26 + sh->slice_qp_delta;
788     if (sh->slice_qp > 51 ||
789         sh->slice_qp < -s->ps.sps->qp_bd_offset) {
790         av_log(s->avctx, AV_LOG_ERROR,
791                "The slice_qp %d is outside the valid range "
792                "[%d, 51].\n",
793                sh->slice_qp,
794                -s->ps.sps->qp_bd_offset);
795         return AVERROR_INVALIDDATA;
796     }
797
798     sh->slice_ctb_addr_rs = sh->slice_segment_addr;
799
800     if (!s->sh.slice_ctb_addr_rs && s->sh.dependent_slice_segment_flag) {
801         av_log(s->avctx, AV_LOG_ERROR, "Impossible slice segment.\n");
802         return AVERROR_INVALIDDATA;
803     }
804
805     s->HEVClc.first_qp_group = !s->sh.dependent_slice_segment_flag;
806
807     if (!s->ps.pps->cu_qp_delta_enabled_flag)
808         s->HEVClc.qp_y = FFUMOD(s->sh.slice_qp + 52 + 2 * s->ps.sps->qp_bd_offset,
809                                 52 + s->ps.sps->qp_bd_offset) - s->ps.sps->qp_bd_offset;
810
811     s->slice_initialized = 1;
812
813     return 0;
814 }
815
816 #define CTB(tab, x, y) ((tab)[(y) * s->ps.sps->ctb_width + (x)])
817
818 #define SET_SAO(elem, value)                            \
819 do {                                                    \
820     if (!sao_merge_up_flag && !sao_merge_left_flag)     \
821         sao->elem = value;                              \
822     else if (sao_merge_left_flag)                       \
823         sao->elem = CTB(s->sao, rx-1, ry).elem;         \
824     else if (sao_merge_up_flag)                         \
825         sao->elem = CTB(s->sao, rx, ry-1).elem;         \
826     else                                                \
827         sao->elem = 0;                                  \
828 } while (0)
829
830 static void hls_sao_param(HEVCContext *s, int rx, int ry)
831 {
832     HEVCLocalContext *lc    = &s->HEVClc;
833     int sao_merge_left_flag = 0;
834     int sao_merge_up_flag   = 0;
835     int shift               = s->ps.sps->bit_depth - FFMIN(s->ps.sps->bit_depth, 10);
836     SAOParams *sao          = &CTB(s->sao, rx, ry);
837     int c_idx, i;
838
839     if (s->sh.slice_sample_adaptive_offset_flag[0] ||
840         s->sh.slice_sample_adaptive_offset_flag[1]) {
841         if (rx > 0) {
842             if (lc->ctb_left_flag)
843                 sao_merge_left_flag = ff_hevc_sao_merge_flag_decode(s);
844         }
845         if (ry > 0 && !sao_merge_left_flag) {
846             if (lc->ctb_up_flag)
847                 sao_merge_up_flag = ff_hevc_sao_merge_flag_decode(s);
848         }
849     }
850
851     for (c_idx = 0; c_idx < 3; c_idx++) {
852         if (!s->sh.slice_sample_adaptive_offset_flag[c_idx]) {
853             sao->type_idx[c_idx] = SAO_NOT_APPLIED;
854             continue;
855         }
856
857         if (c_idx == 2) {
858             sao->type_idx[2] = sao->type_idx[1];
859             sao->eo_class[2] = sao->eo_class[1];
860         } else {
861             SET_SAO(type_idx[c_idx], ff_hevc_sao_type_idx_decode(s));
862         }
863
864         if (sao->type_idx[c_idx] == SAO_NOT_APPLIED)
865             continue;
866
867         for (i = 0; i < 4; i++)
868             SET_SAO(offset_abs[c_idx][i], ff_hevc_sao_offset_abs_decode(s));
869
870         if (sao->type_idx[c_idx] == SAO_BAND) {
871             for (i = 0; i < 4; i++) {
872                 if (sao->offset_abs[c_idx][i]) {
873                     SET_SAO(offset_sign[c_idx][i],
874                             ff_hevc_sao_offset_sign_decode(s));
875                 } else {
876                     sao->offset_sign[c_idx][i] = 0;
877                 }
878             }
879             SET_SAO(band_position[c_idx], ff_hevc_sao_band_position_decode(s));
880         } else if (c_idx != 2) {
881             SET_SAO(eo_class[c_idx], ff_hevc_sao_eo_class_decode(s));
882         }
883
884         // Inferred parameters
885         sao->offset_val[c_idx][0] = 0;
886         for (i = 0; i < 4; i++) {
887             sao->offset_val[c_idx][i + 1] = sao->offset_abs[c_idx][i] << shift;
888             if (sao->type_idx[c_idx] == SAO_EDGE) {
889                 if (i > 1)
890                     sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
891             } else if (sao->offset_sign[c_idx][i]) {
892                 sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
893             }
894         }
895     }
896 }
897
898 #undef SET_SAO
899 #undef CTB
900
901 static void hls_residual_coding(HEVCContext *s, int x0, int y0,
902                                 int log2_trafo_size, enum ScanType scan_idx,
903                                 int c_idx)
904 {
905 #define GET_COORD(offset, n)                                    \
906     do {                                                        \
907         x_c = (scan_x_cg[offset >> 4] << 2) + scan_x_off[n];    \
908         y_c = (scan_y_cg[offset >> 4] << 2) + scan_y_off[n];    \
909     } while (0)
910     HEVCLocalContext *lc    = &s->HEVClc;
911     int transform_skip_flag = 0;
912
913     int last_significant_coeff_x, last_significant_coeff_y;
914     int last_scan_pos;
915     int n_end;
916     int num_coeff    = 0;
917     int greater1_ctx = 1;
918
919     int num_last_subset;
920     int x_cg_last_sig, y_cg_last_sig;
921
922     const uint8_t *scan_x_cg, *scan_y_cg, *scan_x_off, *scan_y_off;
923
924     ptrdiff_t stride = s->frame->linesize[c_idx];
925     int hshift       = s->ps.sps->hshift[c_idx];
926     int vshift       = s->ps.sps->vshift[c_idx];
927     uint8_t *dst     = &s->frame->data[c_idx][(y0 >> vshift) * stride +
928                                               ((x0 >> hshift) << s->ps.sps->pixel_shift)];
929     LOCAL_ALIGNED_32(int16_t, coeffs, [MAX_TB_SIZE * MAX_TB_SIZE]);
930     LOCAL_ALIGNED_8(uint8_t, significant_coeff_group_flag, [8], [8]);
931
932     int trafo_size = 1 << log2_trafo_size;
933     int i, qp, shift, add, scale, scale_m;
934     static const uint8_t level_scale[] = { 40, 45, 51, 57, 64, 72 };
935     const uint8_t *scale_matrix;
936     uint8_t dc_scale;
937
938     memset(coeffs, 0, sizeof(int16_t) * MAX_TB_SIZE * MAX_TB_SIZE);
939     memset(significant_coeff_group_flag, 0, sizeof(uint8_t) * 8 * 8);
940     // Derive QP for dequant
941     if (!lc->cu.cu_transquant_bypass_flag) {
942         static const int qp_c[] = {
943             29, 30, 31, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37
944         };
945
946         static const uint8_t rem6[51 + 2 * 6 + 1] = {
947             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
948             3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
949             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
950         };
951
952         static const uint8_t div6[51 + 2 * 6 + 1] = {
953             0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,  3,  3,  3,
954             3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6,  6,  6,  6,
955             7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10,
956         };
957         int qp_y = lc->qp_y;
958
959         if (c_idx == 0) {
960             qp = qp_y + s->ps.sps->qp_bd_offset;
961         } else {
962             int qp_i, offset;
963
964             if (c_idx == 1)
965                 offset = s->ps.pps->cb_qp_offset + s->sh.slice_cb_qp_offset;
966             else
967                 offset = s->ps.pps->cr_qp_offset + s->sh.slice_cr_qp_offset;
968
969             qp_i = av_clip(qp_y + offset, -s->ps.sps->qp_bd_offset, 57);
970             if (qp_i < 30)
971                 qp = qp_i;
972             else if (qp_i > 43)
973                 qp = qp_i - 6;
974             else
975                 qp = qp_c[qp_i - 30];
976
977             qp += s->ps.sps->qp_bd_offset;
978         }
979
980         shift    = s->ps.sps->bit_depth + log2_trafo_size - 5;
981         add      = 1 << (shift - 1);
982         scale    = level_scale[rem6[qp]] << (div6[qp]);
983         scale_m  = 16; // default when no custom scaling lists.
984         dc_scale = 16;
985
986         if (s->ps.sps->scaling_list_enable_flag) {
987             const ScalingList *sl = s->ps.pps->scaling_list_data_present_flag ?
988                                     &s->ps.pps->scaling_list : &s->ps.sps->scaling_list;
989             int matrix_id = lc->cu.pred_mode != MODE_INTRA;
990
991             if (log2_trafo_size != 5)
992                 matrix_id = 3 * matrix_id + c_idx;
993
994             scale_matrix = sl->sl[log2_trafo_size - 2][matrix_id];
995             if (log2_trafo_size >= 4)
996                 dc_scale = sl->sl_dc[log2_trafo_size - 4][matrix_id];
997         }
998     }
999
1000     if (s->ps.pps->transform_skip_enabled_flag &&
1001         !lc->cu.cu_transquant_bypass_flag   &&
1002         log2_trafo_size == 2) {
1003         transform_skip_flag = ff_hevc_transform_skip_flag_decode(s, c_idx);
1004     }
1005
1006     last_significant_coeff_x =
1007         ff_hevc_last_significant_coeff_x_prefix_decode(s, c_idx, log2_trafo_size);
1008     last_significant_coeff_y =
1009         ff_hevc_last_significant_coeff_y_prefix_decode(s, c_idx, log2_trafo_size);
1010
1011     if (last_significant_coeff_x > 3) {
1012         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_x);
1013         last_significant_coeff_x = (1 << ((last_significant_coeff_x >> 1) - 1)) *
1014                                    (2 + (last_significant_coeff_x & 1)) +
1015                                    suffix;
1016     }
1017
1018     if (last_significant_coeff_y > 3) {
1019         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_y);
1020         last_significant_coeff_y = (1 << ((last_significant_coeff_y >> 1) - 1)) *
1021                                    (2 + (last_significant_coeff_y & 1)) +
1022                                    suffix;
1023     }
1024
1025     if (scan_idx == SCAN_VERT)
1026         FFSWAP(int, last_significant_coeff_x, last_significant_coeff_y);
1027
1028     x_cg_last_sig = last_significant_coeff_x >> 2;
1029     y_cg_last_sig = last_significant_coeff_y >> 2;
1030
1031     switch (scan_idx) {
1032     case SCAN_DIAG: {
1033         int last_x_c = last_significant_coeff_x & 3;
1034         int last_y_c = last_significant_coeff_y & 3;
1035
1036         scan_x_off = ff_hevc_diag_scan4x4_x;
1037         scan_y_off = ff_hevc_diag_scan4x4_y;
1038         num_coeff  = diag_scan4x4_inv[last_y_c][last_x_c];
1039         if (trafo_size == 4) {
1040             scan_x_cg = scan_1x1;
1041             scan_y_cg = scan_1x1;
1042         } else if (trafo_size == 8) {
1043             num_coeff += diag_scan2x2_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1044             scan_x_cg  = diag_scan2x2_x;
1045             scan_y_cg  = diag_scan2x2_y;
1046         } else if (trafo_size == 16) {
1047             num_coeff += diag_scan4x4_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1048             scan_x_cg  = ff_hevc_diag_scan4x4_x;
1049             scan_y_cg  = ff_hevc_diag_scan4x4_y;
1050         } else { // trafo_size == 32
1051             num_coeff += diag_scan8x8_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1052             scan_x_cg  = ff_hevc_diag_scan8x8_x;
1053             scan_y_cg  = ff_hevc_diag_scan8x8_y;
1054         }
1055         break;
1056     }
1057     case SCAN_HORIZ:
1058         scan_x_cg  = horiz_scan2x2_x;
1059         scan_y_cg  = horiz_scan2x2_y;
1060         scan_x_off = horiz_scan4x4_x;
1061         scan_y_off = horiz_scan4x4_y;
1062         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_y][last_significant_coeff_x];
1063         break;
1064     default: //SCAN_VERT
1065         scan_x_cg  = horiz_scan2x2_y;
1066         scan_y_cg  = horiz_scan2x2_x;
1067         scan_x_off = horiz_scan4x4_y;
1068         scan_y_off = horiz_scan4x4_x;
1069         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_x][last_significant_coeff_y];
1070         break;
1071     }
1072     num_coeff++;
1073     num_last_subset = (num_coeff - 1) >> 4;
1074
1075     for (i = num_last_subset; i >= 0; i--) {
1076         int n, m;
1077         int x_cg, y_cg, x_c, y_c;
1078         int implicit_non_zero_coeff = 0;
1079         int64_t trans_coeff_level;
1080         int prev_sig = 0;
1081         int offset   = i << 4;
1082
1083         uint8_t significant_coeff_flag_idx[16];
1084         uint8_t nb_significant_coeff_flag = 0;
1085
1086         x_cg = scan_x_cg[i];
1087         y_cg = scan_y_cg[i];
1088
1089         if (i < num_last_subset && i > 0) {
1090             int ctx_cg = 0;
1091             if (x_cg < (1 << (log2_trafo_size - 2)) - 1)
1092                 ctx_cg += significant_coeff_group_flag[x_cg + 1][y_cg];
1093             if (y_cg < (1 << (log2_trafo_size - 2)) - 1)
1094                 ctx_cg += significant_coeff_group_flag[x_cg][y_cg + 1];
1095
1096             significant_coeff_group_flag[x_cg][y_cg] =
1097                 ff_hevc_significant_coeff_group_flag_decode(s, c_idx, ctx_cg);
1098             implicit_non_zero_coeff = 1;
1099         } else {
1100             significant_coeff_group_flag[x_cg][y_cg] =
1101                 ((x_cg == x_cg_last_sig && y_cg == y_cg_last_sig) ||
1102                  (x_cg == 0 && y_cg == 0));
1103         }
1104
1105         last_scan_pos = num_coeff - offset - 1;
1106
1107         if (i == num_last_subset) {
1108             n_end                         = last_scan_pos - 1;
1109             significant_coeff_flag_idx[0] = last_scan_pos;
1110             nb_significant_coeff_flag     = 1;
1111         } else {
1112             n_end = 15;
1113         }
1114
1115         if (x_cg < ((1 << log2_trafo_size) - 1) >> 2)
1116             prev_sig = significant_coeff_group_flag[x_cg + 1][y_cg];
1117         if (y_cg < ((1 << log2_trafo_size) - 1) >> 2)
1118             prev_sig += significant_coeff_group_flag[x_cg][y_cg + 1] << 1;
1119
1120         for (n = n_end; n >= 0; n--) {
1121             GET_COORD(offset, n);
1122
1123             if (significant_coeff_group_flag[x_cg][y_cg] &&
1124                 (n > 0 || implicit_non_zero_coeff == 0)) {
1125                 if (ff_hevc_significant_coeff_flag_decode(s, c_idx, x_c, y_c,
1126                                                           log2_trafo_size,
1127                                                           scan_idx,
1128                                                           prev_sig) == 1) {
1129                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1130                     nb_significant_coeff_flag++;
1131                     implicit_non_zero_coeff = 0;
1132                 }
1133             } else {
1134                 int last_cg = (x_c == (x_cg << 2) && y_c == (y_cg << 2));
1135                 if (last_cg && implicit_non_zero_coeff && significant_coeff_group_flag[x_cg][y_cg]) {
1136                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1137                     nb_significant_coeff_flag++;
1138                 }
1139             }
1140         }
1141
1142         n_end = nb_significant_coeff_flag;
1143
1144         if (n_end) {
1145             int first_nz_pos_in_cg = 16;
1146             int last_nz_pos_in_cg = -1;
1147             int c_rice_param = 0;
1148             int first_greater1_coeff_idx = -1;
1149             uint8_t coeff_abs_level_greater1_flag[16] = { 0 };
1150             uint16_t coeff_sign_flag;
1151             int sum_abs = 0;
1152             int sign_hidden = 0;
1153
1154             // initialize first elem of coeff_bas_level_greater1_flag
1155             int ctx_set = (i > 0 && c_idx == 0) ? 2 : 0;
1156
1157             if (!(i == num_last_subset) && greater1_ctx == 0)
1158                 ctx_set++;
1159             greater1_ctx      = 1;
1160             last_nz_pos_in_cg = significant_coeff_flag_idx[0];
1161
1162             for (m = 0; m < (n_end > 8 ? 8 : n_end); m++) {
1163                 int n_idx = significant_coeff_flag_idx[m];
1164                 int inc   = (ctx_set << 2) + greater1_ctx;
1165                 coeff_abs_level_greater1_flag[n_idx] =
1166                     ff_hevc_coeff_abs_level_greater1_flag_decode(s, c_idx, inc);
1167                 if (coeff_abs_level_greater1_flag[n_idx]) {
1168                     greater1_ctx = 0;
1169                 } else if (greater1_ctx > 0 && greater1_ctx < 3) {
1170                     greater1_ctx++;
1171                 }
1172
1173                 if (coeff_abs_level_greater1_flag[n_idx] &&
1174                     first_greater1_coeff_idx == -1)
1175                     first_greater1_coeff_idx = n_idx;
1176             }
1177             first_nz_pos_in_cg = significant_coeff_flag_idx[n_end - 1];
1178             sign_hidden        = last_nz_pos_in_cg - first_nz_pos_in_cg >= 4 &&
1179                                  !lc->cu.cu_transquant_bypass_flag;
1180
1181             if (first_greater1_coeff_idx != -1) {
1182                 coeff_abs_level_greater1_flag[first_greater1_coeff_idx] += ff_hevc_coeff_abs_level_greater2_flag_decode(s, c_idx, ctx_set);
1183             }
1184             if (!s->ps.pps->sign_data_hiding_flag || !sign_hidden) {
1185                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag) << (16 - nb_significant_coeff_flag);
1186             } else {
1187                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag - 1) << (16 - (nb_significant_coeff_flag - 1));
1188             }
1189
1190             for (m = 0; m < n_end; m++) {
1191                 n = significant_coeff_flag_idx[m];
1192                 GET_COORD(offset, n);
1193                 trans_coeff_level = 1 + coeff_abs_level_greater1_flag[n];
1194                 if (trans_coeff_level == ((m < 8) ?
1195                                           ((n == first_greater1_coeff_idx) ? 3 : 2) : 1)) {
1196                     trans_coeff_level += ff_hevc_coeff_abs_level_remaining(s, trans_coeff_level, c_rice_param);
1197                     if ((trans_coeff_level) > (3 * (1 << c_rice_param)))
1198                         c_rice_param = FFMIN(c_rice_param + 1, 4);
1199                 }
1200                 if (s->ps.pps->sign_data_hiding_flag && sign_hidden) {
1201                     sum_abs += trans_coeff_level;
1202                     if (n == first_nz_pos_in_cg && ((sum_abs & 1) == 1))
1203                         trans_coeff_level = -trans_coeff_level;
1204                 }
1205                 if (coeff_sign_flag >> 15)
1206                     trans_coeff_level = -trans_coeff_level;
1207                 coeff_sign_flag <<= 1;
1208                 if (!lc->cu.cu_transquant_bypass_flag) {
1209                     if (s->ps.sps->scaling_list_enable_flag) {
1210                         if (y_c || x_c || log2_trafo_size < 4) {
1211                             int pos;
1212                             switch (log2_trafo_size) {
1213                             case 3:  pos = (y_c        << 3) +  x_c;       break;
1214                             case 4:  pos = ((y_c >> 1) << 3) + (x_c >> 1); break;
1215                             case 5:  pos = ((y_c >> 2) << 3) + (x_c >> 2); break;
1216                             default: pos = (y_c        << 2) +  x_c;
1217                             }
1218                             scale_m = scale_matrix[pos];
1219                         } else {
1220                             scale_m = dc_scale;
1221                         }
1222                     }
1223                     trans_coeff_level = (trans_coeff_level * (int64_t)scale * (int64_t)scale_m + add) >> shift;
1224                     if(trans_coeff_level < 0) {
1225                         if((~trans_coeff_level) & 0xFffffffffff8000)
1226                             trans_coeff_level = -32768;
1227                     } else {
1228                         if (trans_coeff_level & 0xffffffffffff8000)
1229                             trans_coeff_level = 32767;
1230                     }
1231                 }
1232                 coeffs[y_c * trafo_size + x_c] = trans_coeff_level;
1233             }
1234         }
1235     }
1236
1237     if (!lc->cu.cu_transquant_bypass_flag) {
1238         if (transform_skip_flag)
1239             s->hevcdsp.dequant(coeffs);
1240         else if (lc->cu.pred_mode == MODE_INTRA && c_idx == 0 &&
1241                  log2_trafo_size == 2)
1242             s->hevcdsp.transform_4x4_luma(coeffs);
1243         else {
1244             int max_xy = FFMAX(last_significant_coeff_x, last_significant_coeff_y);
1245             if (max_xy == 0)
1246                 s->hevcdsp.idct_dc[log2_trafo_size - 2](coeffs);
1247             else {
1248                 int col_limit = last_significant_coeff_x + last_significant_coeff_y + 4;
1249                 if (max_xy < 4)
1250                     col_limit = FFMIN(4, col_limit);
1251                 else if (max_xy < 8)
1252                     col_limit = FFMIN(8, col_limit);
1253                 else if (max_xy < 12)
1254                     col_limit = FFMIN(24, col_limit);
1255                 s->hevcdsp.idct[log2_trafo_size - 2](coeffs, col_limit);
1256             }
1257         }
1258     }
1259     s->hevcdsp.add_residual[log2_trafo_size - 2](dst, coeffs, stride);
1260 }
1261
1262 static int hls_transform_unit(HEVCContext *s, int x0, int y0,
1263                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1264                               int log2_cb_size, int log2_trafo_size,
1265                               int blk_idx, int cbf_luma, int cbf_cb, int cbf_cr)
1266 {
1267     HEVCLocalContext *lc = &s->HEVClc;
1268
1269     if (lc->cu.pred_mode == MODE_INTRA) {
1270         int trafo_size = 1 << log2_trafo_size;
1271         ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1272
1273         s->hpc.intra_pred[log2_trafo_size - 2](s, x0, y0, 0);
1274         if (log2_trafo_size > 2) {
1275             trafo_size = trafo_size << (s->ps.sps->hshift[1] - 1);
1276             ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1277             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 1);
1278             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 2);
1279         } else if (blk_idx == 3) {
1280             trafo_size = trafo_size << s->ps.sps->hshift[1];
1281             ff_hevc_set_neighbour_available(s, xBase, yBase,
1282                                             trafo_size, trafo_size);
1283             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 1);
1284             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 2);
1285         }
1286     }
1287
1288     if (cbf_luma || cbf_cb || cbf_cr) {
1289         int scan_idx   = SCAN_DIAG;
1290         int scan_idx_c = SCAN_DIAG;
1291
1292         if (s->ps.pps->cu_qp_delta_enabled_flag && !lc->tu.is_cu_qp_delta_coded) {
1293             lc->tu.cu_qp_delta = ff_hevc_cu_qp_delta_abs(s);
1294             if (lc->tu.cu_qp_delta != 0)
1295                 if (ff_hevc_cu_qp_delta_sign_flag(s) == 1)
1296                     lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta;
1297             lc->tu.is_cu_qp_delta_coded = 1;
1298
1299             if (lc->tu.cu_qp_delta < -(26 + s->ps.sps->qp_bd_offset / 2) ||
1300                 lc->tu.cu_qp_delta >  (25 + s->ps.sps->qp_bd_offset / 2)) {
1301                 av_log(s->avctx, AV_LOG_ERROR,
1302                        "The cu_qp_delta %d is outside the valid range "
1303                        "[%d, %d].\n",
1304                        lc->tu.cu_qp_delta,
1305                        -(26 + s->ps.sps->qp_bd_offset / 2),
1306                         (25 + s->ps.sps->qp_bd_offset / 2));
1307                 return AVERROR_INVALIDDATA;
1308             }
1309
1310             ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size);
1311         }
1312
1313         if (lc->cu.pred_mode == MODE_INTRA && log2_trafo_size < 4) {
1314             if (lc->tu.cur_intra_pred_mode >= 6 &&
1315                 lc->tu.cur_intra_pred_mode <= 14) {
1316                 scan_idx = SCAN_VERT;
1317             } else if (lc->tu.cur_intra_pred_mode >= 22 &&
1318                        lc->tu.cur_intra_pred_mode <= 30) {
1319                 scan_idx = SCAN_HORIZ;
1320             }
1321
1322             if (lc->pu.intra_pred_mode_c >=  6 &&
1323                 lc->pu.intra_pred_mode_c <= 14) {
1324                 scan_idx_c = SCAN_VERT;
1325             } else if (lc->pu.intra_pred_mode_c >= 22 &&
1326                        lc->pu.intra_pred_mode_c <= 30) {
1327                 scan_idx_c = SCAN_HORIZ;
1328             }
1329         }
1330
1331         if (cbf_luma)
1332             hls_residual_coding(s, x0, y0, log2_trafo_size, scan_idx, 0);
1333         if (log2_trafo_size > 2) {
1334             if (cbf_cb)
1335                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 1);
1336             if (cbf_cr)
1337                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 2);
1338         } else if (blk_idx == 3) {
1339             if (cbf_cb)
1340                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 1);
1341             if (cbf_cr)
1342                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2);
1343         }
1344     }
1345     return 0;
1346 }
1347
1348 static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size)
1349 {
1350     int cb_size          = 1 << log2_cb_size;
1351     int log2_min_pu_size = s->ps.sps->log2_min_pu_size;
1352
1353     int min_pu_width     = s->ps.sps->min_pu_width;
1354     int x_end = FFMIN(x0 + cb_size, s->ps.sps->width);
1355     int y_end = FFMIN(y0 + cb_size, s->ps.sps->height);
1356     int i, j;
1357
1358     for (j = (y0 >> log2_min_pu_size); j < (y_end >> log2_min_pu_size); j++)
1359         for (i = (x0 >> log2_min_pu_size); i < (x_end >> log2_min_pu_size); i++)
1360             s->is_pcm[i + j * min_pu_width] = 2;
1361 }
1362
1363 static int hls_transform_tree(HEVCContext *s, int x0, int y0,
1364                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1365                               int log2_cb_size, int log2_trafo_size,
1366                               int trafo_depth, int blk_idx,
1367                               int cbf_cb, int cbf_cr)
1368 {
1369     HEVCLocalContext *lc = &s->HEVClc;
1370     uint8_t split_transform_flag;
1371     int ret;
1372
1373     if (lc->cu.intra_split_flag) {
1374         if (trafo_depth == 1)
1375             lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[blk_idx];
1376     } else {
1377         lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[0];
1378     }
1379
1380     if (log2_trafo_size <= s->ps.sps->log2_max_trafo_size &&
1381         log2_trafo_size >  s->ps.sps->log2_min_tb_size    &&
1382         trafo_depth     < lc->cu.max_trafo_depth       &&
1383         !(lc->cu.intra_split_flag && trafo_depth == 0)) {
1384         split_transform_flag = ff_hevc_split_transform_flag_decode(s, log2_trafo_size);
1385     } else {
1386         int inter_split = s->ps.sps->max_transform_hierarchy_depth_inter == 0 &&
1387                           lc->cu.pred_mode == MODE_INTER &&
1388                           lc->cu.part_mode != PART_2Nx2N &&
1389                           trafo_depth == 0;
1390
1391         split_transform_flag = log2_trafo_size > s->ps.sps->log2_max_trafo_size ||
1392                                (lc->cu.intra_split_flag && trafo_depth == 0) ||
1393                                inter_split;
1394     }
1395
1396     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cb))
1397         cbf_cb = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1398     else if (log2_trafo_size > 2 || trafo_depth == 0)
1399         cbf_cb = 0;
1400     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cr))
1401         cbf_cr = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1402     else if (log2_trafo_size > 2 || trafo_depth == 0)
1403         cbf_cr = 0;
1404
1405     if (split_transform_flag) {
1406         const int trafo_size_split = 1 << (log2_trafo_size - 1);
1407         const int x1 = x0 + trafo_size_split;
1408         const int y1 = y0 + trafo_size_split;
1409
1410 #define SUBDIVIDE(x, y, idx)                                                    \
1411 do {                                                                            \
1412     ret = hls_transform_tree(s, x, y, x0, y0, cb_xBase, cb_yBase, log2_cb_size, \
1413                              log2_trafo_size - 1, trafo_depth + 1, idx,         \
1414                              cbf_cb, cbf_cr);                                   \
1415     if (ret < 0)                                                                \
1416         return ret;                                                             \
1417 } while (0)
1418
1419         SUBDIVIDE(x0, y0, 0);
1420         SUBDIVIDE(x1, y0, 1);
1421         SUBDIVIDE(x0, y1, 2);
1422         SUBDIVIDE(x1, y1, 3);
1423
1424 #undef SUBDIVIDE
1425     } else {
1426         int min_tu_size      = 1 << s->ps.sps->log2_min_tb_size;
1427         int log2_min_tu_size = s->ps.sps->log2_min_tb_size;
1428         int min_tu_width     = s->ps.sps->min_tb_width;
1429         int cbf_luma         = 1;
1430
1431         if (lc->cu.pred_mode == MODE_INTRA || trafo_depth != 0 ||
1432             cbf_cb || cbf_cr)
1433             cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth);
1434
1435         ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase,
1436                                  log2_cb_size, log2_trafo_size,
1437                                  blk_idx, cbf_luma, cbf_cb, cbf_cr);
1438         if (ret < 0)
1439             return ret;
1440         // TODO: store cbf_luma somewhere else
1441         if (cbf_luma) {
1442             int i, j;
1443             for (i = 0; i < (1 << log2_trafo_size); i += min_tu_size)
1444                 for (j = 0; j < (1 << log2_trafo_size); j += min_tu_size) {
1445                     int x_tu = (x0 + j) >> log2_min_tu_size;
1446                     int y_tu = (y0 + i) >> log2_min_tu_size;
1447                     s->cbf_luma[y_tu * min_tu_width + x_tu] = 1;
1448                 }
1449         }
1450         if (!s->sh.disable_deblocking_filter_flag) {
1451             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_trafo_size);
1452             if (s->ps.pps->transquant_bypass_enable_flag &&
1453                 lc->cu.cu_transquant_bypass_flag)
1454                 set_deblocking_bypass(s, x0, y0, log2_trafo_size);
1455         }
1456     }
1457     return 0;
1458 }
1459
1460 static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size)
1461 {
1462     //TODO: non-4:2:0 support
1463     HEVCLocalContext *lc = &s->HEVClc;
1464     GetBitContext gb;
1465     int cb_size   = 1 << log2_cb_size;
1466     ptrdiff_t stride0 = s->frame->linesize[0];
1467     ptrdiff_t stride1 = s->frame->linesize[1];
1468     ptrdiff_t stride2 = s->frame->linesize[2];
1469     uint8_t *dst0 = &s->frame->data[0][y0 * stride0 + (x0 << s->ps.sps->pixel_shift)];
1470     uint8_t *dst1 = &s->frame->data[1][(y0 >> s->ps.sps->vshift[1]) * stride1 + ((x0 >> s->ps.sps->hshift[1]) << s->ps.sps->pixel_shift)];
1471     uint8_t *dst2 = &s->frame->data[2][(y0 >> s->ps.sps->vshift[2]) * stride2 + ((x0 >> s->ps.sps->hshift[2]) << s->ps.sps->pixel_shift)];
1472
1473     int length         = cb_size * cb_size * s->ps.sps->pcm.bit_depth + ((cb_size * cb_size) >> 1) * s->ps.sps->pcm.bit_depth_chroma;
1474     const uint8_t *pcm = skip_bytes(&lc->cc, (length + 7) >> 3);
1475     int ret;
1476
1477     if (!s->sh.disable_deblocking_filter_flag)
1478         ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
1479
1480     ret = init_get_bits(&gb, pcm, length);
1481     if (ret < 0)
1482         return ret;
1483
1484     s->hevcdsp.put_pcm(dst0, stride0, cb_size,     &gb, s->ps.sps->pcm.bit_depth);
1485     s->hevcdsp.put_pcm(dst1, stride1, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1486     s->hevcdsp.put_pcm(dst2, stride2, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1487     return 0;
1488 }
1489
1490 static void hls_mvd_coding(HEVCContext *s, int x0, int y0, int log2_cb_size)
1491 {
1492     HEVCLocalContext *lc = &s->HEVClc;
1493     int x = ff_hevc_abs_mvd_greater0_flag_decode(s);
1494     int y = ff_hevc_abs_mvd_greater0_flag_decode(s);
1495
1496     if (x)
1497         x += ff_hevc_abs_mvd_greater1_flag_decode(s);
1498     if (y)
1499         y += ff_hevc_abs_mvd_greater1_flag_decode(s);
1500
1501     switch (x) {
1502     case 2: lc->pu.mvd.x = ff_hevc_mvd_decode(s);           break;
1503     case 1: lc->pu.mvd.x = ff_hevc_mvd_sign_flag_decode(s); break;
1504     case 0: lc->pu.mvd.x = 0;                               break;
1505     }
1506
1507     switch (y) {
1508     case 2: lc->pu.mvd.y = ff_hevc_mvd_decode(s);           break;
1509     case 1: lc->pu.mvd.y = ff_hevc_mvd_sign_flag_decode(s); break;
1510     case 0: lc->pu.mvd.y = 0;                               break;
1511     }
1512 }
1513
1514 /**
1515  * 8.5.3.2.2.1 Luma sample interpolation process
1516  *
1517  * @param s HEVC decoding context
1518  * @param dst target buffer for block data at block position
1519  * @param dststride stride of the dst buffer
1520  * @param ref reference picture buffer at origin (0, 0)
1521  * @param mv motion vector (relative to block position) to get pixel data from
1522  * @param x_off horizontal position of block from origin (0, 0)
1523  * @param y_off vertical position of block from origin (0, 0)
1524  * @param block_w width of block
1525  * @param block_h height of block
1526  */
1527 static void luma_mc(HEVCContext *s, int16_t *dst, ptrdiff_t dststride,
1528                     AVFrame *ref, const Mv *mv, int x_off, int y_off,
1529                     int block_w, int block_h, int pred_idx)
1530 {
1531     HEVCLocalContext *lc = &s->HEVClc;
1532     uint8_t *src         = ref->data[0];
1533     ptrdiff_t srcstride  = ref->linesize[0];
1534     int pic_width        = s->ps.sps->width;
1535     int pic_height       = s->ps.sps->height;
1536
1537     int mx         = mv->x & 3;
1538     int my         = mv->y & 3;
1539     int extra_left = ff_hevc_qpel_extra_before[mx];
1540     int extra_top  = ff_hevc_qpel_extra_before[my];
1541
1542     x_off += mv->x >> 2;
1543     y_off += mv->y >> 2;
1544     src   += y_off * srcstride + (x_off * (1 << s->ps.sps->pixel_shift));
1545
1546     if (x_off < extra_left || y_off < extra_top ||
1547         x_off >= pic_width - block_w - ff_hevc_qpel_extra_after[mx] ||
1548         y_off >= pic_height - block_h - ff_hevc_qpel_extra_after[my]) {
1549         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1550         int offset = extra_top * srcstride + (extra_left << s->ps.sps->pixel_shift);
1551         int buf_offset = extra_top *
1552                          edge_emu_stride + (extra_left << s->ps.sps->pixel_shift);
1553
1554         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src - offset,
1555                                  edge_emu_stride, srcstride,
1556                                  block_w + ff_hevc_qpel_extra[mx],
1557                                  block_h + ff_hevc_qpel_extra[my],
1558                                  x_off - extra_left, y_off - extra_top,
1559                                  pic_width, pic_height);
1560         src = lc->edge_emu_buffer + buf_offset;
1561         srcstride = edge_emu_stride;
1562     }
1563     s->hevcdsp.put_hevc_qpel[!!my][!!mx][pred_idx](dst, dststride, src, srcstride,
1564                                                    block_h, mx, my, lc->mc_buffer);
1565 }
1566
1567 /**
1568  * 8.5.3.2.2.2 Chroma sample interpolation process
1569  *
1570  * @param s HEVC decoding context
1571  * @param dst1 target buffer for block data at block position (U plane)
1572  * @param dst2 target buffer for block data at block position (V plane)
1573  * @param dststride stride of the dst1 and dst2 buffers
1574  * @param ref reference picture buffer at origin (0, 0)
1575  * @param mv motion vector (relative to block position) to get pixel data from
1576  * @param x_off horizontal position of block from origin (0, 0)
1577  * @param y_off vertical position of block from origin (0, 0)
1578  * @param block_w width of block
1579  * @param block_h height of block
1580  */
1581 static void chroma_mc(HEVCContext *s, int16_t *dst1, int16_t *dst2,
1582                       ptrdiff_t dststride, AVFrame *ref, const Mv *mv,
1583                       int x_off, int y_off, int block_w, int block_h, int pred_idx)
1584 {
1585     HEVCLocalContext *lc = &s->HEVClc;
1586     uint8_t *src1        = ref->data[1];
1587     uint8_t *src2        = ref->data[2];
1588     ptrdiff_t src1stride = ref->linesize[1];
1589     ptrdiff_t src2stride = ref->linesize[2];
1590     int pic_width        = s->ps.sps->width >> 1;
1591     int pic_height       = s->ps.sps->height >> 1;
1592
1593     int mx = mv->x & 7;
1594     int my = mv->y & 7;
1595
1596     x_off += mv->x >> 3;
1597     y_off += mv->y >> 3;
1598     src1  += y_off * src1stride + (x_off * (1 << s->ps.sps->pixel_shift));
1599     src2  += y_off * src2stride + (x_off * (1 << s->ps.sps->pixel_shift));
1600
1601     if (x_off < EPEL_EXTRA_BEFORE || y_off < EPEL_EXTRA_AFTER ||
1602         x_off >= pic_width - block_w - EPEL_EXTRA_AFTER ||
1603         y_off >= pic_height - block_h - EPEL_EXTRA_AFTER) {
1604         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1605         int offset1 = EPEL_EXTRA_BEFORE * (src1stride + (1 << s->ps.sps->pixel_shift));
1606         int buf_offset1 = EPEL_EXTRA_BEFORE *
1607                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1608         int offset2 = EPEL_EXTRA_BEFORE * (src2stride + (1 << s->ps.sps->pixel_shift));
1609         int buf_offset2 = EPEL_EXTRA_BEFORE *
1610                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1611
1612         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src1 - offset1,
1613                                  edge_emu_stride, src1stride,
1614                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1615                                  x_off - EPEL_EXTRA_BEFORE,
1616                                  y_off - EPEL_EXTRA_BEFORE,
1617                                  pic_width, pic_height);
1618
1619         src1 = lc->edge_emu_buffer + buf_offset1;
1620         src1stride = edge_emu_stride;
1621         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1622                                                        block_h, mx, my, lc->mc_buffer);
1623
1624         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src2 - offset2,
1625                                  edge_emu_stride, src2stride,
1626                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1627                                  x_off - EPEL_EXTRA_BEFORE,
1628                                  y_off - EPEL_EXTRA_BEFORE,
1629                                  pic_width, pic_height);
1630         src2 = lc->edge_emu_buffer + buf_offset2;
1631         src2stride = edge_emu_stride;
1632
1633         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1634                                                        block_h, mx, my, lc->mc_buffer);
1635     } else {
1636         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1637                                                        block_h, mx, my, lc->mc_buffer);
1638         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1639                                                        block_h, mx, my, lc->mc_buffer);
1640     }
1641 }
1642
1643 static void hevc_await_progress(HEVCContext *s, HEVCFrame *ref,
1644                                 const Mv *mv, int y0, int height)
1645 {
1646     int y = (mv->y >> 2) + y0 + height + 9;
1647     ff_thread_await_progress(&ref->tf, y, 0);
1648 }
1649
1650 static void hevc_luma_mv_mpv_mode(HEVCContext *s, int x0, int y0, int nPbW,
1651                                   int nPbH, int log2_cb_size, int part_idx,
1652                                   int merge_idx, MvField *mv)
1653 {
1654     HEVCLocalContext *lc             = &s->HEVClc;
1655     enum InterPredIdc inter_pred_idc = PRED_L0;
1656     int mvp_flag;
1657
1658     ff_hevc_set_neighbour_available(s, x0, y0, nPbW, nPbH);
1659     if (s->sh.slice_type == HEVC_SLICE_B)
1660         inter_pred_idc = ff_hevc_inter_pred_idc_decode(s, nPbW, nPbH);
1661
1662     if (inter_pred_idc != PRED_L1) {
1663         if (s->sh.nb_refs[L0])
1664             mv->ref_idx[0]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L0]);
1665
1666         mv->pred_flag[0] = 1;
1667         hls_mvd_coding(s, x0, y0, 0);
1668         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1669         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1670                                  part_idx, merge_idx, mv, mvp_flag, 0);
1671         mv->mv[0].x += lc->pu.mvd.x;
1672         mv->mv[0].y += lc->pu.mvd.y;
1673     }
1674
1675     if (inter_pred_idc != PRED_L0) {
1676         if (s->sh.nb_refs[L1])
1677             mv->ref_idx[1]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L1]);
1678
1679         if (s->sh.mvd_l1_zero_flag == 1 && inter_pred_idc == PRED_BI) {
1680             AV_ZERO32(&lc->pu.mvd);
1681         } else {
1682             hls_mvd_coding(s, x0, y0, 1);
1683         }
1684
1685         mv->pred_flag[1] = 1;
1686         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1687         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1688                                  part_idx, merge_idx, mv, mvp_flag, 1);
1689         mv->mv[1].x += lc->pu.mvd.x;
1690         mv->mv[1].y += lc->pu.mvd.y;
1691     }
1692 }
1693
1694 static void hls_prediction_unit(HEVCContext *s, int x0, int y0,
1695                                 int nPbW, int nPbH,
1696                                 int log2_cb_size, int partIdx)
1697 {
1698     static const int pred_indices[] = {
1699         [4] = 0, [8] = 1, [12] = 2, [16] = 3, [24] = 4, [32] = 5, [48] = 6, [64] = 7,
1700     };
1701     const int pred_idx = pred_indices[nPbW];
1702
1703 #define POS(c_idx, x, y)                                                              \
1704     &s->frame->data[c_idx][((y) >> s->ps.sps->vshift[c_idx]) * s->frame->linesize[c_idx] + \
1705                            (((x) >> s->ps.sps->hshift[c_idx]) << s->ps.sps->pixel_shift)]
1706     HEVCLocalContext *lc = &s->HEVClc;
1707     int merge_idx = 0;
1708     struct MvField current_mv = {{{ 0 }}};
1709
1710     int min_pu_width = s->ps.sps->min_pu_width;
1711     int weighted_pred = (s->sh.slice_type == HEVC_SLICE_P && s->ps.pps->weighted_pred_flag) ||
1712                         (s->sh.slice_type == HEVC_SLICE_B && s->ps.pps->weighted_bipred_flag);
1713
1714     MvField *tab_mvf = s->ref->tab_mvf;
1715     RefPicList  *refPicList = s->ref->refPicList;
1716     HEVCFrame *ref0, *ref1;
1717
1718     ptrdiff_t tmpstride = MAX_PB_SIZE * sizeof(int16_t);
1719
1720     uint8_t *dst0 = POS(0, x0, y0);
1721     uint8_t *dst1 = POS(1, x0, y0);
1722     uint8_t *dst2 = POS(2, x0, y0);
1723     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
1724     int min_cb_width     = s->ps.sps->min_cb_width;
1725     int x_cb             = x0 >> log2_min_cb_size;
1726     int y_cb             = y0 >> log2_min_cb_size;
1727     int x_pu, y_pu;
1728     int i, j;
1729
1730     int skip_flag = SAMPLE_CTB(s->skip_flag, x_cb, y_cb);
1731
1732     if (!skip_flag)
1733         lc->pu.merge_flag = ff_hevc_merge_flag_decode(s);
1734
1735     if (skip_flag || lc->pu.merge_flag) {
1736         if (s->sh.max_num_merge_cand > 1)
1737             merge_idx = ff_hevc_merge_idx_decode(s);
1738         else
1739             merge_idx = 0;
1740
1741         ff_hevc_luma_mv_merge_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1742                                    partIdx, merge_idx, &current_mv);
1743     } else {
1744         hevc_luma_mv_mpv_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1745                               partIdx, merge_idx, &current_mv);
1746     }
1747
1748     x_pu = x0 >> s->ps.sps->log2_min_pu_size;
1749     y_pu = y0 >> s->ps.sps->log2_min_pu_size;
1750
1751     for (j = 0; j < nPbH >> s->ps.sps->log2_min_pu_size; j++)
1752         for (i = 0; i < nPbW >> s->ps.sps->log2_min_pu_size; i++)
1753             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i] = current_mv;
1754
1755     if (current_mv.pred_flag[0]) {
1756         ref0 = refPicList[0].ref[current_mv.ref_idx[0]];
1757         if (!ref0)
1758             return;
1759         hevc_await_progress(s, ref0, &current_mv.mv[0], y0, nPbH);
1760     }
1761     if (current_mv.pred_flag[1]) {
1762         ref1 = refPicList[1].ref[current_mv.ref_idx[1]];
1763         if (!ref1)
1764             return;
1765         hevc_await_progress(s, ref1, &current_mv.mv[1], y0, nPbH);
1766     }
1767
1768     if (current_mv.pred_flag[0] && !current_mv.pred_flag[1]) {
1769         LOCAL_ALIGNED_16(int16_t,  tmp, [MAX_PB_SIZE * MAX_PB_SIZE]);
1770         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1771
1772         luma_mc(s, tmp, tmpstride, ref0->frame,
1773                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1774
1775         if (weighted_pred) {
1776             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1777                                                s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1778                                                s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1779                                                dst0, s->frame->linesize[0], tmp,
1780                                                tmpstride, nPbH);
1781         } else {
1782             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1783         }
1784         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1785                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1786
1787         if (weighted_pred) {
1788             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1789                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1790                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1791                                                       dst1, s->frame->linesize[1], tmp, tmpstride,
1792                                                       nPbH / 2);
1793             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1794                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1795                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1796                                                       dst2, s->frame->linesize[2], tmp2, tmpstride,
1797                                                       nPbH / 2);
1798         } else {
1799             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1800             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1801         }
1802     } else if (!current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1803         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1804         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1805
1806         luma_mc(s, tmp, tmpstride, ref1->frame,
1807                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1808
1809         if (weighted_pred) {
1810             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1811                                                s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1812                                                s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1813                                                dst0, s->frame->linesize[0], tmp, tmpstride,
1814                                                nPbH);
1815         } else {
1816             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1817         }
1818
1819         chroma_mc(s, tmp, tmp2, tmpstride, ref1->frame,
1820                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1821
1822         if (weighted_pred) {
1823             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1824                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1825                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1826                                                       dst1, s->frame->linesize[1], tmp, tmpstride, nPbH/2);
1827             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1828                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1829                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1830                                                       dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH/2);
1831         } else {
1832             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1833             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1834         }
1835     } else if (current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1836         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1837         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1838         LOCAL_ALIGNED_16(int16_t, tmp3, [MAX_PB_SIZE * MAX_PB_SIZE]);
1839         LOCAL_ALIGNED_16(int16_t, tmp4, [MAX_PB_SIZE * MAX_PB_SIZE]);
1840
1841         luma_mc(s, tmp, tmpstride, ref0->frame,
1842                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1843         luma_mc(s, tmp2, tmpstride, ref1->frame,
1844                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1845
1846         if (weighted_pred) {
1847             s->hevcdsp.weighted_pred_avg[pred_idx](s->sh.luma_log2_weight_denom,
1848                                                    s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1849                                                    s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1850                                                    s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1851                                                    s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1852                                                    dst0, s->frame->linesize[0],
1853                                                    tmp, tmp2, tmpstride, nPbH);
1854         } else {
1855             s->hevcdsp.put_unweighted_pred_avg[pred_idx](dst0, s->frame->linesize[0],
1856                                                          tmp, tmp2, tmpstride, nPbH);
1857         }
1858
1859         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1860                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1861         chroma_mc(s, tmp3, tmp4, tmpstride, ref1->frame,
1862                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1863
1864         if (weighted_pred) {
1865             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1866                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1867                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1868                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1869                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1870                                                           dst1, s->frame->linesize[1], tmp, tmp3,
1871                                                           tmpstride, nPbH / 2);
1872             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1873                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1874                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1875                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1876                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1877                                                           dst2, s->frame->linesize[2], tmp2, tmp4,
1878                                                           tmpstride, nPbH / 2);
1879         } else {
1880             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst1, s->frame->linesize[1], tmp, tmp3,  tmpstride, nPbH/2);
1881             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmp4, tmpstride, nPbH/2);
1882         }
1883     }
1884 }
1885
1886 /**
1887  * 8.4.1
1888  */
1889 static int luma_intra_pred_mode(HEVCContext *s, int x0, int y0, int pu_size,
1890                                 int prev_intra_luma_pred_flag)
1891 {
1892     HEVCLocalContext *lc = &s->HEVClc;
1893     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
1894     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
1895     int min_pu_width     = s->ps.sps->min_pu_width;
1896     int size_in_pus      = pu_size >> s->ps.sps->log2_min_pu_size;
1897     int x0b              = x0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1898     int y0b              = y0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1899
1900     int cand_up   = (lc->ctb_up_flag || y0b) ?
1901                     s->tab_ipm[(y_pu - 1) * min_pu_width + x_pu] : INTRA_DC;
1902     int cand_left = (lc->ctb_left_flag || x0b) ?
1903                     s->tab_ipm[y_pu * min_pu_width + x_pu - 1]   : INTRA_DC;
1904
1905     int y_ctb = (y0 >> (s->ps.sps->log2_ctb_size)) << (s->ps.sps->log2_ctb_size);
1906
1907     MvField *tab_mvf = s->ref->tab_mvf;
1908     int intra_pred_mode;
1909     int candidate[3];
1910     int i, j;
1911
1912     // intra_pred_mode prediction does not cross vertical CTB boundaries
1913     if ((y0 - 1) < y_ctb)
1914         cand_up = INTRA_DC;
1915
1916     if (cand_left == cand_up) {
1917         if (cand_left < 2) {
1918             candidate[0] = INTRA_PLANAR;
1919             candidate[1] = INTRA_DC;
1920             candidate[2] = INTRA_ANGULAR_26;
1921         } else {
1922             candidate[0] = cand_left;
1923             candidate[1] = 2 + ((cand_left - 2 - 1 + 32) & 31);
1924             candidate[2] = 2 + ((cand_left - 2 + 1) & 31);
1925         }
1926     } else {
1927         candidate[0] = cand_left;
1928         candidate[1] = cand_up;
1929         if (candidate[0] != INTRA_PLANAR && candidate[1] != INTRA_PLANAR) {
1930             candidate[2] = INTRA_PLANAR;
1931         } else if (candidate[0] != INTRA_DC && candidate[1] != INTRA_DC) {
1932             candidate[2] = INTRA_DC;
1933         } else {
1934             candidate[2] = INTRA_ANGULAR_26;
1935         }
1936     }
1937
1938     if (prev_intra_luma_pred_flag) {
1939         intra_pred_mode = candidate[lc->pu.mpm_idx];
1940     } else {
1941         if (candidate[0] > candidate[1])
1942             FFSWAP(uint8_t, candidate[0], candidate[1]);
1943         if (candidate[0] > candidate[2])
1944             FFSWAP(uint8_t, candidate[0], candidate[2]);
1945         if (candidate[1] > candidate[2])
1946             FFSWAP(uint8_t, candidate[1], candidate[2]);
1947
1948         intra_pred_mode = lc->pu.rem_intra_luma_pred_mode;
1949         for (i = 0; i < 3; i++)
1950             if (intra_pred_mode >= candidate[i])
1951                 intra_pred_mode++;
1952     }
1953
1954     /* write the intra prediction units into the mv array */
1955     if (!size_in_pus)
1956         size_in_pus = 1;
1957     for (i = 0; i < size_in_pus; i++) {
1958         memset(&s->tab_ipm[(y_pu + i) * min_pu_width + x_pu],
1959                intra_pred_mode, size_in_pus);
1960
1961         for (j = 0; j < size_in_pus; j++) {
1962             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].is_intra     = 1;
1963             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[0] = 0;
1964             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[1] = 0;
1965             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[0]   = 0;
1966             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[1]   = 0;
1967             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].x      = 0;
1968             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].y      = 0;
1969             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].x      = 0;
1970             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].y      = 0;
1971         }
1972     }
1973
1974     return intra_pred_mode;
1975 }
1976
1977 static av_always_inline void set_ct_depth(HEVCContext *s, int x0, int y0,
1978                                           int log2_cb_size, int ct_depth)
1979 {
1980     int length = (1 << log2_cb_size) >> s->ps.sps->log2_min_cb_size;
1981     int x_cb   = x0 >> s->ps.sps->log2_min_cb_size;
1982     int y_cb   = y0 >> s->ps.sps->log2_min_cb_size;
1983     int y;
1984
1985     for (y = 0; y < length; y++)
1986         memset(&s->tab_ct_depth[(y_cb + y) * s->ps.sps->min_cb_width + x_cb],
1987                ct_depth, length);
1988 }
1989
1990 static void intra_prediction_unit(HEVCContext *s, int x0, int y0,
1991                                   int log2_cb_size)
1992 {
1993     HEVCLocalContext *lc = &s->HEVClc;
1994     static const uint8_t intra_chroma_table[4] = { 0, 26, 10, 1 };
1995     uint8_t prev_intra_luma_pred_flag[4];
1996     int split   = lc->cu.part_mode == PART_NxN;
1997     int pb_size = (1 << log2_cb_size) >> split;
1998     int side    = split + 1;
1999     int chroma_mode;
2000     int i, j;
2001
2002     for (i = 0; i < side; i++)
2003         for (j = 0; j < side; j++)
2004             prev_intra_luma_pred_flag[2 * i + j] = ff_hevc_prev_intra_luma_pred_flag_decode(s);
2005
2006     for (i = 0; i < side; i++) {
2007         for (j = 0; j < side; j++) {
2008             if (prev_intra_luma_pred_flag[2 * i + j])
2009                 lc->pu.mpm_idx = ff_hevc_mpm_idx_decode(s);
2010             else
2011                 lc->pu.rem_intra_luma_pred_mode = ff_hevc_rem_intra_luma_pred_mode_decode(s);
2012
2013             lc->pu.intra_pred_mode[2 * i + j] =
2014                 luma_intra_pred_mode(s, x0 + pb_size * j, y0 + pb_size * i, pb_size,
2015                                      prev_intra_luma_pred_flag[2 * i + j]);
2016         }
2017     }
2018
2019     chroma_mode = ff_hevc_intra_chroma_pred_mode_decode(s);
2020     if (chroma_mode != 4) {
2021         if (lc->pu.intra_pred_mode[0] == intra_chroma_table[chroma_mode])
2022             lc->pu.intra_pred_mode_c = 34;
2023         else
2024             lc->pu.intra_pred_mode_c = intra_chroma_table[chroma_mode];
2025     } else {
2026         lc->pu.intra_pred_mode_c = lc->pu.intra_pred_mode[0];
2027     }
2028 }
2029
2030 static void intra_prediction_unit_default_value(HEVCContext *s,
2031                                                 int x0, int y0,
2032                                                 int log2_cb_size)
2033 {
2034     HEVCLocalContext *lc = &s->HEVClc;
2035     int pb_size          = 1 << log2_cb_size;
2036     int size_in_pus      = pb_size >> s->ps.sps->log2_min_pu_size;
2037     int min_pu_width     = s->ps.sps->min_pu_width;
2038     MvField *tab_mvf     = s->ref->tab_mvf;
2039     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
2040     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
2041     int j, k;
2042
2043     if (size_in_pus == 0)
2044         size_in_pus = 1;
2045     for (j = 0; j < size_in_pus; j++) {
2046         memset(&s->tab_ipm[(y_pu + j) * min_pu_width + x_pu], INTRA_DC, size_in_pus);
2047         for (k = 0; k < size_in_pus; k++)
2048             tab_mvf[(y_pu + j) * min_pu_width + x_pu + k].is_intra = lc->cu.pred_mode == MODE_INTRA;
2049     }
2050 }
2051
2052 static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
2053 {
2054     int cb_size          = 1 << log2_cb_size;
2055     HEVCLocalContext *lc = &s->HEVClc;
2056     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
2057     int length           = cb_size >> log2_min_cb_size;
2058     int min_cb_width     = s->ps.sps->min_cb_width;
2059     int x_cb             = x0 >> log2_min_cb_size;
2060     int y_cb             = y0 >> log2_min_cb_size;
2061     int x, y, ret;
2062
2063     lc->cu.x                = x0;
2064     lc->cu.y                = y0;
2065     lc->cu.pred_mode        = MODE_INTRA;
2066     lc->cu.part_mode        = PART_2Nx2N;
2067     lc->cu.intra_split_flag = 0;
2068
2069     SAMPLE_CTB(s->skip_flag, x_cb, y_cb) = 0;
2070     for (x = 0; x < 4; x++)
2071         lc->pu.intra_pred_mode[x] = 1;
2072     if (s->ps.pps->transquant_bypass_enable_flag) {
2073         lc->cu.cu_transquant_bypass_flag = ff_hevc_cu_transquant_bypass_flag_decode(s);
2074         if (lc->cu.cu_transquant_bypass_flag)
2075             set_deblocking_bypass(s, x0, y0, log2_cb_size);
2076     } else
2077         lc->cu.cu_transquant_bypass_flag = 0;
2078
2079     if (s->sh.slice_type != HEVC_SLICE_I) {
2080         uint8_t skip_flag = ff_hevc_skip_flag_decode(s, x0, y0, x_cb, y_cb);
2081
2082         x = y_cb * min_cb_width + x_cb;
2083         for (y = 0; y < length; y++) {
2084             memset(&s->skip_flag[x], skip_flag, length);
2085             x += min_cb_width;
2086         }
2087         lc->cu.pred_mode = skip_flag ? MODE_SKIP : MODE_INTER;
2088     }
2089
2090     if (SAMPLE_CTB(s->skip_flag, x_cb, y_cb)) {
2091         hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2092         intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2093
2094         if (!s->sh.disable_deblocking_filter_flag)
2095             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2096     } else {
2097         int pcm_flag = 0;
2098
2099         if (s->sh.slice_type != HEVC_SLICE_I)
2100             lc->cu.pred_mode = ff_hevc_pred_mode_decode(s);
2101         if (lc->cu.pred_mode != MODE_INTRA ||
2102             log2_cb_size == s->ps.sps->log2_min_cb_size) {
2103             lc->cu.part_mode        = ff_hevc_part_mode_decode(s, log2_cb_size);
2104             lc->cu.intra_split_flag = lc->cu.part_mode == PART_NxN &&
2105                                       lc->cu.pred_mode == MODE_INTRA;
2106         }
2107
2108         if (lc->cu.pred_mode == MODE_INTRA) {
2109             if (lc->cu.part_mode == PART_2Nx2N && s->ps.sps->pcm_enabled_flag &&
2110                 log2_cb_size >= s->ps.sps->pcm.log2_min_pcm_cb_size &&
2111                 log2_cb_size <= s->ps.sps->pcm.log2_max_pcm_cb_size) {
2112                 pcm_flag = ff_hevc_pcm_flag_decode(s);
2113             }
2114             if (pcm_flag) {
2115                 intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2116                 ret = hls_pcm_sample(s, x0, y0, log2_cb_size);
2117                 if (s->ps.sps->pcm.loop_filter_disable_flag)
2118                     set_deblocking_bypass(s, x0, y0, log2_cb_size);
2119
2120                 if (ret < 0)
2121                     return ret;
2122             } else {
2123                 intra_prediction_unit(s, x0, y0, log2_cb_size);
2124             }
2125         } else {
2126             intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2127             switch (lc->cu.part_mode) {
2128             case PART_2Nx2N:
2129                 hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2130                 break;
2131             case PART_2NxN:
2132                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size / 2, log2_cb_size, 0);
2133                 hls_prediction_unit(s, x0, y0 + cb_size / 2, cb_size, cb_size / 2, log2_cb_size, 1);
2134                 break;
2135             case PART_Nx2N:
2136                 hls_prediction_unit(s, x0,               y0, cb_size / 2, cb_size, log2_cb_size, 0);
2137                 hls_prediction_unit(s, x0 + cb_size / 2, y0, cb_size / 2, cb_size, log2_cb_size, 1);
2138                 break;
2139             case PART_2NxnU:
2140                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size     / 4, log2_cb_size, 0);
2141                 hls_prediction_unit(s, x0, y0 + cb_size / 4, cb_size, cb_size * 3 / 4, log2_cb_size, 1);
2142                 break;
2143             case PART_2NxnD:
2144                 hls_prediction_unit(s, x0, y0,                   cb_size, cb_size * 3 / 4, log2_cb_size, 0);
2145                 hls_prediction_unit(s, x0, y0 + cb_size * 3 / 4, cb_size, cb_size     / 4, log2_cb_size, 1);
2146                 break;
2147             case PART_nLx2N:
2148                 hls_prediction_unit(s, x0,               y0, cb_size     / 4, cb_size, log2_cb_size, 0);
2149                 hls_prediction_unit(s, x0 + cb_size / 4, y0, cb_size * 3 / 4, cb_size, log2_cb_size, 1);
2150                 break;
2151             case PART_nRx2N:
2152                 hls_prediction_unit(s, x0,                   y0, cb_size * 3 / 4, cb_size, log2_cb_size, 0);
2153                 hls_prediction_unit(s, x0 + cb_size * 3 / 4, y0, cb_size     / 4, cb_size, log2_cb_size, 1);
2154                 break;
2155             case PART_NxN:
2156                 hls_prediction_unit(s, x0,               y0,               cb_size / 2, cb_size / 2, log2_cb_size, 0);
2157                 hls_prediction_unit(s, x0 + cb_size / 2, y0,               cb_size / 2, cb_size / 2, log2_cb_size, 1);
2158                 hls_prediction_unit(s, x0,               y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 2);
2159                 hls_prediction_unit(s, x0 + cb_size / 2, y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 3);
2160                 break;
2161             }
2162         }
2163
2164         if (!pcm_flag) {
2165             int rqt_root_cbf = 1;
2166
2167             if (lc->cu.pred_mode != MODE_INTRA &&
2168                 !(lc->cu.part_mode == PART_2Nx2N && lc->pu.merge_flag)) {
2169                 rqt_root_cbf = ff_hevc_no_residual_syntax_flag_decode(s);
2170             }
2171             if (rqt_root_cbf) {
2172                 lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ?
2173                                          s->ps.sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag :
2174                                          s->ps.sps->max_transform_hierarchy_depth_inter;
2175                 ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0,
2176                                          log2_cb_size,
2177                                          log2_cb_size, 0, 0, 0, 0);
2178                 if (ret < 0)
2179                     return ret;
2180             } else {
2181                 if (!s->sh.disable_deblocking_filter_flag)
2182                     ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2183             }
2184         }
2185     }
2186
2187     if (s->ps.pps->cu_qp_delta_enabled_flag && lc->tu.is_cu_qp_delta_coded == 0)
2188         ff_hevc_set_qPy(s, x0, y0, x0, y0, log2_cb_size);
2189
2190     x = y_cb * min_cb_width + x_cb;
2191     for (y = 0; y < length; y++) {
2192         memset(&s->qp_y_tab[x], lc->qp_y, length);
2193         x += min_cb_width;
2194     }
2195
2196     set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth);
2197
2198     return 0;
2199 }
2200
2201 static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
2202                                int log2_cb_size, int cb_depth)
2203 {
2204     HEVCLocalContext *lc = &s->HEVClc;
2205     const int cb_size    = 1 << log2_cb_size;
2206     int split_cu;
2207
2208     lc->ct.depth = cb_depth;
2209     if (x0 + cb_size <= s->ps.sps->width  &&
2210         y0 + cb_size <= s->ps.sps->height &&
2211         log2_cb_size > s->ps.sps->log2_min_cb_size) {
2212         split_cu = ff_hevc_split_coding_unit_flag_decode(s, cb_depth, x0, y0);
2213     } else {
2214         split_cu = (log2_cb_size > s->ps.sps->log2_min_cb_size);
2215     }
2216     if (s->ps.pps->cu_qp_delta_enabled_flag &&
2217         log2_cb_size >= s->ps.sps->log2_ctb_size - s->ps.pps->diff_cu_qp_delta_depth) {
2218         lc->tu.is_cu_qp_delta_coded = 0;
2219         lc->tu.cu_qp_delta          = 0;
2220     }
2221
2222     if (split_cu) {
2223         const int cb_size_split = cb_size >> 1;
2224         const int x1 = x0 + cb_size_split;
2225         const int y1 = y0 + cb_size_split;
2226
2227         log2_cb_size--;
2228         cb_depth++;
2229
2230 #define SUBDIVIDE(x, y)                                                \
2231 do {                                                                   \
2232     if (x < s->ps.sps->width && y < s->ps.sps->height) {                     \
2233         int ret = hls_coding_quadtree(s, x, y, log2_cb_size, cb_depth);\
2234         if (ret < 0)                                                   \
2235             return ret;                                                \
2236     }                                                                  \
2237 } while (0)
2238
2239         SUBDIVIDE(x0, y0);
2240         SUBDIVIDE(x1, y0);
2241         SUBDIVIDE(x0, y1);
2242         SUBDIVIDE(x1, y1);
2243     } else {
2244         int ret = hls_coding_unit(s, x0, y0, log2_cb_size);
2245         if (ret < 0)
2246             return ret;
2247     }
2248
2249     return 0;
2250 }
2251
2252 static void hls_decode_neighbour(HEVCContext *s, int x_ctb, int y_ctb,
2253                                  int ctb_addr_ts)
2254 {
2255     HEVCLocalContext *lc  = &s->HEVClc;
2256     int ctb_size          = 1 << s->ps.sps->log2_ctb_size;
2257     int ctb_addr_rs       = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2258     int ctb_addr_in_slice = ctb_addr_rs - s->sh.slice_addr;
2259
2260     s->tab_slice_address[ctb_addr_rs] = s->sh.slice_addr;
2261
2262     if (s->ps.pps->entropy_coding_sync_enabled_flag) {
2263         if (x_ctb == 0 && (y_ctb & (ctb_size - 1)) == 0)
2264             lc->first_qp_group = 1;
2265         lc->end_of_tiles_x = s->ps.sps->width;
2266     } else if (s->ps.pps->tiles_enabled_flag) {
2267         if (ctb_addr_ts && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[ctb_addr_ts - 1]) {
2268             int idxX = s->ps.pps->col_idxX[x_ctb >> s->ps.sps->log2_ctb_size];
2269             lc->start_of_tiles_x = x_ctb;
2270             lc->end_of_tiles_x   = x_ctb + (s->ps.pps->column_width[idxX] << s->ps.sps->log2_ctb_size);
2271             lc->first_qp_group   = 1;
2272         }
2273     } else {
2274         lc->end_of_tiles_x = s->ps.sps->width;
2275     }
2276
2277     lc->end_of_tiles_y = FFMIN(y_ctb + ctb_size, s->ps.sps->height);
2278
2279     lc->boundary_flags = 0;
2280     if (s->ps.pps->tiles_enabled_flag) {
2281         if (x_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - 1]])
2282             lc->boundary_flags |= BOUNDARY_LEFT_TILE;
2283         if (x_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - 1])
2284             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2285         if (y_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - s->ps.sps->ctb_width]])
2286             lc->boundary_flags |= BOUNDARY_UPPER_TILE;
2287         if (y_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - s->ps.sps->ctb_width])
2288             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2289     } else {
2290         if (!ctb_addr_in_slice)
2291             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2292         if (ctb_addr_in_slice < s->ps.sps->ctb_width)
2293             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2294     }
2295
2296     lc->ctb_left_flag = ((x_ctb > 0) && (ctb_addr_in_slice > 0) && !(lc->boundary_flags & BOUNDARY_LEFT_TILE));
2297     lc->ctb_up_flag   = ((y_ctb > 0) && (ctb_addr_in_slice >= s->ps.sps->ctb_width) && !(lc->boundary_flags & BOUNDARY_UPPER_TILE));
2298     lc->ctb_up_right_flag = ((y_ctb > 0)  && (ctb_addr_in_slice+1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs+1 - s->ps.sps->ctb_width]]));
2299     lc->ctb_up_left_flag = ((x_ctb > 0) && (y_ctb > 0)  && (ctb_addr_in_slice-1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs-1 - s->ps.sps->ctb_width]]));
2300 }
2301
2302 static int hls_slice_data(HEVCContext *s)
2303 {
2304     int ctb_size    = 1 << s->ps.sps->log2_ctb_size;
2305     int more_data   = 1;
2306     int x_ctb       = 0;
2307     int y_ctb       = 0;
2308     int ctb_addr_ts = s->ps.pps->ctb_addr_rs_to_ts[s->sh.slice_ctb_addr_rs];
2309     int ret;
2310
2311     while (more_data && ctb_addr_ts < s->ps.sps->ctb_size) {
2312         int ctb_addr_rs = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2313
2314         x_ctb = (ctb_addr_rs % ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2315         y_ctb = (ctb_addr_rs / ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2316         hls_decode_neighbour(s, x_ctb, y_ctb, ctb_addr_ts);
2317
2318         ff_hevc_cabac_init(s, ctb_addr_ts);
2319
2320         hls_sao_param(s, x_ctb >> s->ps.sps->log2_ctb_size, y_ctb >> s->ps.sps->log2_ctb_size);
2321
2322         s->deblock[ctb_addr_rs].beta_offset = s->sh.beta_offset;
2323         s->deblock[ctb_addr_rs].tc_offset   = s->sh.tc_offset;
2324         s->filter_slice_edges[ctb_addr_rs]  = s->sh.slice_loop_filter_across_slices_enabled_flag;
2325
2326         ret = hls_coding_quadtree(s, x_ctb, y_ctb, s->ps.sps->log2_ctb_size, 0);
2327         if (ret < 0)
2328             return ret;
2329         more_data = !ff_hevc_end_of_slice_flag_decode(s);
2330
2331         ctb_addr_ts++;
2332         ff_hevc_save_states(s, ctb_addr_ts);
2333         ff_hevc_hls_filters(s, x_ctb, y_ctb, ctb_size);
2334     }
2335
2336     if (x_ctb + ctb_size >= s->ps.sps->width &&
2337         y_ctb + ctb_size >= s->ps.sps->height)
2338         ff_hevc_hls_filter(s, x_ctb, y_ctb);
2339
2340     return ctb_addr_ts;
2341 }
2342
2343 static void restore_tqb_pixels(HEVCContext *s)
2344 {
2345     int min_pu_size = 1 << s->ps.sps->log2_min_pu_size;
2346     int x, y, c_idx;
2347
2348     for (c_idx = 0; c_idx < 3; c_idx++) {
2349         ptrdiff_t stride = s->frame->linesize[c_idx];
2350         int hshift       = s->ps.sps->hshift[c_idx];
2351         int vshift       = s->ps.sps->vshift[c_idx];
2352         for (y = 0; y < s->ps.sps->min_pu_height; y++) {
2353             for (x = 0; x < s->ps.sps->min_pu_width; x++) {
2354                 if (s->is_pcm[y * s->ps.sps->min_pu_width + x]) {
2355                     int n;
2356                     int len      = min_pu_size >> hshift;
2357                     uint8_t *src = &s->frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2358                     uint8_t *dst = &s->sao_frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2359                     for (n = 0; n < (min_pu_size >> vshift); n++) {
2360                         memcpy(dst, src, len);
2361                         src += stride;
2362                         dst += stride;
2363                     }
2364                 }
2365             }
2366         }
2367     }
2368 }
2369
2370 static int set_side_data(HEVCContext *s)
2371 {
2372     AVFrame *out = s->ref->frame;
2373
2374     if (s->sei.frame_packing.present &&
2375         s->sei.frame_packing.arrangement_type >= 3 &&
2376         s->sei.frame_packing.arrangement_type <= 5 &&
2377         s->sei.frame_packing.content_interpretation_type > 0 &&
2378         s->sei.frame_packing.content_interpretation_type < 3) {
2379         AVStereo3D *stereo = av_stereo3d_create_side_data(out);
2380         if (!stereo)
2381             return AVERROR(ENOMEM);
2382
2383         switch (s->sei.frame_packing.arrangement_type) {
2384         case 3:
2385             if (s->sei.frame_packing.quincunx_subsampling)
2386                 stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
2387             else
2388                 stereo->type = AV_STEREO3D_SIDEBYSIDE;
2389             break;
2390         case 4:
2391             stereo->type = AV_STEREO3D_TOPBOTTOM;
2392             break;
2393         case 5:
2394             stereo->type = AV_STEREO3D_FRAMESEQUENCE;
2395             break;
2396         }
2397
2398         if (s->sei.frame_packing.content_interpretation_type == 2)
2399             stereo->flags = AV_STEREO3D_FLAG_INVERT;
2400     }
2401
2402     if (s->sei.display_orientation.present &&
2403         (s->sei.display_orientation.anticlockwise_rotation ||
2404          s->sei.display_orientation.hflip || s->sei.display_orientation.vflip)) {
2405         double angle = s->sei.display_orientation.anticlockwise_rotation * 360 / (double) (1 << 16);
2406         AVFrameSideData *rotation = av_frame_new_side_data(out,
2407                                                            AV_FRAME_DATA_DISPLAYMATRIX,
2408                                                            sizeof(int32_t) * 9);
2409         if (!rotation)
2410             return AVERROR(ENOMEM);
2411
2412         av_display_rotation_set((int32_t *)rotation->data, angle);
2413         av_display_matrix_flip((int32_t *)rotation->data,
2414                                s->sei.display_orientation.hflip,
2415                                s->sei.display_orientation.vflip);
2416     }
2417
2418     if (s->sei.alternative_transfer.present &&
2419         av_color_transfer_name(s->sei.alternative_transfer.preferred_transfer_characteristics) &&
2420         s->sei.alternative_transfer.preferred_transfer_characteristics != AVCOL_TRC_UNSPECIFIED) {
2421         s->avctx->color_trc = out->color_trc = s->sei.alternative_transfer.preferred_transfer_characteristics;
2422     }
2423
2424     return 0;
2425 }
2426
2427 static int hevc_frame_start(HEVCContext *s)
2428 {
2429     HEVCLocalContext *lc = &s->HEVClc;
2430     int ret;
2431
2432     memset(s->horizontal_bs, 0, 2 * s->bs_width * (s->bs_height + 1));
2433     memset(s->vertical_bs,   0, 2 * s->bs_width * (s->bs_height + 1));
2434     memset(s->cbf_luma,      0, s->ps.sps->min_tb_width * s->ps.sps->min_tb_height);
2435     memset(s->is_pcm,        0, s->ps.sps->min_pu_width * s->ps.sps->min_pu_height);
2436
2437     lc->start_of_tiles_x = 0;
2438     s->is_decoded        = 0;
2439     s->first_nal_type    = s->nal_unit_type;
2440
2441     if (s->ps.pps->tiles_enabled_flag)
2442         lc->end_of_tiles_x = s->ps.pps->column_width[0] << s->ps.sps->log2_ctb_size;
2443
2444     ret = ff_hevc_set_new_ref(s, s->ps.sps->sao_enabled ? &s->sao_frame : &s->frame,
2445                               s->poc);
2446     if (ret < 0)
2447         goto fail;
2448
2449     ret = ff_hevc_frame_rps(s);
2450     if (ret < 0) {
2451         av_log(s->avctx, AV_LOG_ERROR, "Error constructing the frame RPS.\n");
2452         goto fail;
2453     }
2454
2455     s->ref->frame->key_frame = IS_IRAP(s);
2456
2457     ret = set_side_data(s);
2458     if (ret < 0)
2459         goto fail;
2460
2461     av_frame_unref(s->output_frame);
2462     ret = ff_hevc_output_frame(s, s->output_frame, 0);
2463     if (ret < 0)
2464         goto fail;
2465
2466     ff_thread_finish_setup(s->avctx);
2467
2468     return 0;
2469
2470 fail:
2471     if (s->ref)
2472         ff_hevc_unref_frame(s, s->ref, ~0);
2473     s->ref = NULL;
2474     return ret;
2475 }
2476
2477 static int decode_nal_unit(HEVCContext *s, const H2645NAL *nal)
2478 {
2479     HEVCLocalContext *lc = &s->HEVClc;
2480     GetBitContext *gb    = &lc->gb;
2481     int ctb_addr_ts, ret;
2482
2483     *gb              = nal->gb;
2484     s->nal_unit_type = nal->type;
2485     s->temporal_id   = nal->temporal_id;
2486
2487     switch (s->nal_unit_type) {
2488     case HEVC_NAL_VPS:
2489         ret = ff_hevc_decode_nal_vps(gb, s->avctx, &s->ps);
2490         if (ret < 0)
2491             goto fail;
2492         break;
2493     case HEVC_NAL_SPS:
2494         ret = ff_hevc_decode_nal_sps(gb, s->avctx, &s->ps,
2495                                      s->apply_defdispwin);
2496         if (ret < 0)
2497             goto fail;
2498         break;
2499     case HEVC_NAL_PPS:
2500         ret = ff_hevc_decode_nal_pps(gb, s->avctx, &s->ps);
2501         if (ret < 0)
2502             goto fail;
2503         break;
2504     case HEVC_NAL_SEI_PREFIX:
2505     case HEVC_NAL_SEI_SUFFIX:
2506         ret = ff_hevc_decode_nal_sei(gb, s->avctx, &s->sei,
2507                                      s->nal_unit_type);
2508         if (ret < 0)
2509             goto fail;
2510         break;
2511     case HEVC_NAL_TRAIL_R:
2512     case HEVC_NAL_TRAIL_N:
2513     case HEVC_NAL_TSA_N:
2514     case HEVC_NAL_TSA_R:
2515     case HEVC_NAL_STSA_N:
2516     case HEVC_NAL_STSA_R:
2517     case HEVC_NAL_BLA_W_LP:
2518     case HEVC_NAL_BLA_W_RADL:
2519     case HEVC_NAL_BLA_N_LP:
2520     case HEVC_NAL_IDR_W_RADL:
2521     case HEVC_NAL_IDR_N_LP:
2522     case HEVC_NAL_CRA_NUT:
2523     case HEVC_NAL_RADL_N:
2524     case HEVC_NAL_RADL_R:
2525     case HEVC_NAL_RASL_N:
2526     case HEVC_NAL_RASL_R:
2527         ret = hls_slice_header(s);
2528         if (ret < 0)
2529             return ret;
2530
2531         if (s->max_ra == INT_MAX) {
2532             if (s->nal_unit_type == HEVC_NAL_CRA_NUT || IS_BLA(s)) {
2533                 s->max_ra = s->poc;
2534             } else {
2535                 if (IS_IDR(s))
2536                     s->max_ra = INT_MIN;
2537             }
2538         }
2539
2540         if ((s->nal_unit_type == HEVC_NAL_RASL_R || s->nal_unit_type == HEVC_NAL_RASL_N) &&
2541             s->poc <= s->max_ra) {
2542             s->is_decoded = 0;
2543             break;
2544         } else {
2545             if (s->nal_unit_type == HEVC_NAL_RASL_R && s->poc > s->max_ra)
2546                 s->max_ra = INT_MIN;
2547         }
2548
2549         if (s->sh.first_slice_in_pic_flag) {
2550             ret = hevc_frame_start(s);
2551             if (ret < 0)
2552                 return ret;
2553         } else if (!s->ref) {
2554             av_log(s->avctx, AV_LOG_ERROR, "First slice in a frame missing.\n");
2555             goto fail;
2556         }
2557
2558         if (s->nal_unit_type != s->first_nal_type) {
2559             av_log(s->avctx, AV_LOG_ERROR,
2560                    "Non-matching NAL types of the VCL NALUs: %d %d\n",
2561                    s->first_nal_type, s->nal_unit_type);
2562             return AVERROR_INVALIDDATA;
2563         }
2564
2565         if (!s->sh.dependent_slice_segment_flag &&
2566             s->sh.slice_type != HEVC_SLICE_I) {
2567             ret = ff_hevc_slice_rpl(s);
2568             if (ret < 0) {
2569                 av_log(s->avctx, AV_LOG_WARNING,
2570                        "Error constructing the reference lists for the current slice.\n");
2571                 goto fail;
2572             }
2573         }
2574
2575         if (s->sh.first_slice_in_pic_flag && s->avctx->hwaccel) {
2576             ret = s->avctx->hwaccel->start_frame(s->avctx, NULL, 0);
2577             if (ret < 0)
2578                 goto fail;
2579         }
2580
2581         if (s->avctx->hwaccel) {
2582             ret = s->avctx->hwaccel->decode_slice(s->avctx, nal->raw_data, nal->raw_size);
2583             if (ret < 0)
2584                 goto fail;
2585         } else {
2586             ctb_addr_ts = hls_slice_data(s);
2587             if (ctb_addr_ts >= (s->ps.sps->ctb_width * s->ps.sps->ctb_height)) {
2588                 s->is_decoded = 1;
2589                 if ((s->ps.pps->transquant_bypass_enable_flag ||
2590                      (s->ps.sps->pcm.loop_filter_disable_flag && s->ps.sps->pcm_enabled_flag)) &&
2591                     s->ps.sps->sao_enabled)
2592                     restore_tqb_pixels(s);
2593             }
2594
2595             if (ctb_addr_ts < 0) {
2596                 ret = ctb_addr_ts;
2597                 goto fail;
2598             }
2599         }
2600         break;
2601     case HEVC_NAL_EOS_NUT:
2602     case HEVC_NAL_EOB_NUT:
2603         s->seq_decode = (s->seq_decode + 1) & 0xff;
2604         s->max_ra     = INT_MAX;
2605         break;
2606     case HEVC_NAL_AUD:
2607     case HEVC_NAL_FD_NUT:
2608         break;
2609     default:
2610         av_log(s->avctx, AV_LOG_INFO,
2611                "Skipping NAL unit %d\n", s->nal_unit_type);
2612     }
2613
2614     return 0;
2615 fail:
2616     if (s->avctx->err_recognition & AV_EF_EXPLODE)
2617         return ret;
2618     return 0;
2619 }
2620
2621 static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length)
2622 {
2623     int i, ret = 0;
2624
2625     s->ref = NULL;
2626     s->eos = 0;
2627
2628     /* split the input packet into NAL units, so we know the upper bound on the
2629      * number of slices in the frame */
2630     ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx, s->is_nalff,
2631                                 s->nal_length_size, s->avctx->codec_id);
2632     if (ret < 0) {
2633         av_log(s->avctx, AV_LOG_ERROR,
2634                "Error splitting the input into NAL units.\n");
2635         return ret;
2636     }
2637
2638     for (i = 0; i < s->pkt.nb_nals; i++) {
2639         if (s->pkt.nals[i].type == HEVC_NAL_EOB_NUT ||
2640             s->pkt.nals[i].type == HEVC_NAL_EOS_NUT)
2641             s->eos = 1;
2642     }
2643
2644     /* decode the NAL units */
2645     for (i = 0; i < s->pkt.nb_nals; i++) {
2646         ret = decode_nal_unit(s, &s->pkt.nals[i]);
2647         if (ret < 0) {
2648             av_log(s->avctx, AV_LOG_WARNING,
2649                    "Error parsing NAL unit #%d.\n", i);
2650             goto fail;
2651         }
2652     }
2653
2654 fail:
2655     if (s->ref)
2656         ff_thread_report_progress(&s->ref->tf, INT_MAX, 0);
2657
2658     return ret;
2659 }
2660
2661 static void print_md5(void *log_ctx, int level, uint8_t md5[16])
2662 {
2663     int i;
2664     for (i = 0; i < 16; i++)
2665         av_log(log_ctx, level, "%02"PRIx8, md5[i]);
2666 }
2667
2668 static int verify_md5(HEVCContext *s, AVFrame *frame)
2669 {
2670     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
2671     int pixel_shift;
2672     int i, j;
2673
2674     if (!desc)
2675         return AVERROR(EINVAL);
2676
2677     pixel_shift = desc->comp[0].depth > 8;
2678
2679     av_log(s->avctx, AV_LOG_DEBUG, "Verifying checksum for frame with POC %d: ",
2680            s->poc);
2681
2682     /* the checksums are LE, so we have to byteswap for >8bpp formats
2683      * on BE arches */
2684 #if HAVE_BIGENDIAN
2685     if (pixel_shift && !s->checksum_buf) {
2686         av_fast_malloc(&s->checksum_buf, &s->checksum_buf_size,
2687                        FFMAX3(frame->linesize[0], frame->linesize[1],
2688                               frame->linesize[2]));
2689         if (!s->checksum_buf)
2690             return AVERROR(ENOMEM);
2691     }
2692 #endif
2693
2694     for (i = 0; frame->data[i]; i++) {
2695         int width  = s->avctx->coded_width;
2696         int height = s->avctx->coded_height;
2697         int w = (i == 1 || i == 2) ? (width  >> desc->log2_chroma_w) : width;
2698         int h = (i == 1 || i == 2) ? (height >> desc->log2_chroma_h) : height;
2699         uint8_t md5[16];
2700
2701         av_md5_init(s->md5_ctx);
2702         for (j = 0; j < h; j++) {
2703             const uint8_t *src = frame->data[i] + j * frame->linesize[i];
2704 #if HAVE_BIGENDIAN
2705             if (pixel_shift) {
2706                 s->bdsp.bswap16_buf((uint16_t *) s->checksum_buf,
2707                                     (const uint16_t *) src, w);
2708                 src = s->checksum_buf;
2709             }
2710 #endif
2711             av_md5_update(s->md5_ctx, src, w << pixel_shift);
2712         }
2713         av_md5_final(s->md5_ctx, md5);
2714
2715         if (!memcmp(md5, s->sei.picture_hash.md5[i], 16)) {
2716             av_log   (s->avctx, AV_LOG_DEBUG, "plane %d - correct ", i);
2717             print_md5(s->avctx, AV_LOG_DEBUG, md5);
2718             av_log   (s->avctx, AV_LOG_DEBUG, "; ");
2719         } else {
2720             av_log   (s->avctx, AV_LOG_ERROR, "mismatching checksum of plane %d - ", i);
2721             print_md5(s->avctx, AV_LOG_ERROR, md5);
2722             av_log   (s->avctx, AV_LOG_ERROR, " != ");
2723             print_md5(s->avctx, AV_LOG_ERROR, s->sei.picture_hash.md5[i]);
2724             av_log   (s->avctx, AV_LOG_ERROR, "\n");
2725             return AVERROR_INVALIDDATA;
2726         }
2727     }
2728
2729     av_log(s->avctx, AV_LOG_DEBUG, "\n");
2730
2731     return 0;
2732 }
2733
2734 static int hevc_decode_extradata(HEVCContext *s, uint8_t *buf, int length)
2735 {
2736     AVCodecContext *avctx = s->avctx;
2737     GetByteContext gb;
2738     int ret, i;
2739
2740     bytestream2_init(&gb, buf, length);
2741
2742     if (length > 3 && (buf[0] || buf[1] || buf[2] > 1)) {
2743         /* It seems the extradata is encoded as hvcC format.
2744          * Temporarily, we support configurationVersion==0 until 14496-15 3rd
2745          * is finalized. When finalized, configurationVersion will be 1 and we
2746          * can recognize hvcC by checking if avctx->extradata[0]==1 or not. */
2747         int i, j, num_arrays, nal_len_size;
2748
2749         s->is_nalff = 1;
2750
2751         bytestream2_skip(&gb, 21);
2752         nal_len_size = (bytestream2_get_byte(&gb) & 3) + 1;
2753         num_arrays   = bytestream2_get_byte(&gb);
2754
2755         /* nal units in the hvcC always have length coded with 2 bytes,
2756          * so put a fake nal_length_size = 2 while parsing them */
2757         s->nal_length_size = 2;
2758
2759         /* Decode nal units from hvcC. */
2760         for (i = 0; i < num_arrays; i++) {
2761             int type = bytestream2_get_byte(&gb) & 0x3f;
2762             int cnt  = bytestream2_get_be16(&gb);
2763
2764             for (j = 0; j < cnt; j++) {
2765                 // +2 for the nal size field
2766                 int nalsize = bytestream2_peek_be16(&gb) + 2;
2767                 if (bytestream2_get_bytes_left(&gb) < nalsize) {
2768                     av_log(s->avctx, AV_LOG_ERROR,
2769                            "Invalid NAL unit size in extradata.\n");
2770                     return AVERROR_INVALIDDATA;
2771                 }
2772
2773                 ret = decode_nal_units(s, gb.buffer, nalsize);
2774                 if (ret < 0) {
2775                     av_log(avctx, AV_LOG_ERROR,
2776                            "Decoding nal unit %d %d from hvcC failed\n",
2777                            type, i);
2778                     return ret;
2779                 }
2780                 bytestream2_skip(&gb, nalsize);
2781             }
2782         }
2783
2784         /* Now store right nal length size, that will be used to parse
2785          * all other nals */
2786         s->nal_length_size = nal_len_size;
2787     } else {
2788         s->is_nalff = 0;
2789         ret = decode_nal_units(s, buf, length);
2790         if (ret < 0)
2791             return ret;
2792     }
2793
2794     /* export stream parameters from the first SPS */
2795     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
2796         if (s->ps.sps_list[i]) {
2797             const HEVCSPS *sps = (const HEVCSPS*)s->ps.sps_list[i]->data;
2798             export_stream_params(s->avctx, &s->ps, sps);
2799             break;
2800         }
2801     }
2802
2803     return 0;
2804 }
2805
2806 static int hevc_decode_frame(AVCodecContext *avctx, void *data, int *got_output,
2807                              AVPacket *avpkt)
2808 {
2809     int ret;
2810     int new_extradata_size;
2811     uint8_t *new_extradata;
2812     HEVCContext *s = avctx->priv_data;
2813
2814     if (!avpkt->size) {
2815         ret = ff_hevc_output_frame(s, data, 1);
2816         if (ret < 0)
2817             return ret;
2818
2819         *got_output = ret;
2820         return 0;
2821     }
2822
2823     new_extradata = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA,
2824                                             &new_extradata_size);
2825     if (new_extradata && new_extradata_size > 0) {
2826         ret = hevc_decode_extradata(s, new_extradata, new_extradata_size);
2827         if (ret < 0)
2828             return ret;
2829     }
2830
2831     s->ref = NULL;
2832     ret    = decode_nal_units(s, avpkt->data, avpkt->size);
2833     if (ret < 0)
2834         return ret;
2835
2836     if (avctx->hwaccel) {
2837         if (s->ref && avctx->hwaccel->end_frame(avctx) < 0)
2838             av_log(avctx, AV_LOG_ERROR,
2839                    "hardware accelerator failed to decode picture\n");
2840     } else {
2841         /* verify the SEI checksum */
2842         if (avctx->err_recognition & AV_EF_CRCCHECK && s->is_decoded &&
2843             s->sei.picture_hash.is_md5) {
2844             ret = verify_md5(s, s->ref->frame);
2845             if (ret < 0 && avctx->err_recognition & AV_EF_EXPLODE) {
2846                 ff_hevc_unref_frame(s, s->ref, ~0);
2847                 return ret;
2848             }
2849         }
2850     }
2851     s->sei.picture_hash.is_md5 = 0;
2852
2853     if (s->is_decoded) {
2854         av_log(avctx, AV_LOG_DEBUG, "Decoded frame with POC %d.\n", s->poc);
2855         s->is_decoded = 0;
2856     }
2857
2858     if (s->output_frame->buf[0]) {
2859         av_frame_move_ref(data, s->output_frame);
2860         *got_output = 1;
2861     }
2862
2863     return avpkt->size;
2864 }
2865
2866 static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
2867 {
2868     int ret = ff_thread_ref_frame(&dst->tf, &src->tf);
2869     if (ret < 0)
2870         return ret;
2871
2872     dst->tab_mvf_buf = av_buffer_ref(src->tab_mvf_buf);
2873     if (!dst->tab_mvf_buf)
2874         goto fail;
2875     dst->tab_mvf = src->tab_mvf;
2876
2877     dst->rpl_tab_buf = av_buffer_ref(src->rpl_tab_buf);
2878     if (!dst->rpl_tab_buf)
2879         goto fail;
2880     dst->rpl_tab = src->rpl_tab;
2881
2882     dst->rpl_buf = av_buffer_ref(src->rpl_buf);
2883     if (!dst->rpl_buf)
2884         goto fail;
2885
2886     dst->poc        = src->poc;
2887     dst->ctb_count  = src->ctb_count;
2888     dst->flags      = src->flags;
2889     dst->sequence   = src->sequence;
2890
2891     if (src->hwaccel_picture_private) {
2892         dst->hwaccel_priv_buf = av_buffer_ref(src->hwaccel_priv_buf);
2893         if (!dst->hwaccel_priv_buf)
2894             goto fail;
2895         dst->hwaccel_picture_private = dst->hwaccel_priv_buf->data;
2896     }
2897
2898     return 0;
2899 fail:
2900     ff_hevc_unref_frame(s, dst, ~0);
2901     return AVERROR(ENOMEM);
2902 }
2903
2904 static av_cold int hevc_decode_free(AVCodecContext *avctx)
2905 {
2906     HEVCContext       *s = avctx->priv_data;
2907     int i;
2908
2909     pic_arrays_free(s);
2910
2911     av_freep(&s->md5_ctx);
2912
2913     av_frame_free(&s->tmp_frame);
2914     av_frame_free(&s->output_frame);
2915
2916     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2917         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2918         av_frame_free(&s->DPB[i].frame);
2919     }
2920
2921     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++)
2922         av_buffer_unref(&s->ps.vps_list[i]);
2923     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++)
2924         av_buffer_unref(&s->ps.sps_list[i]);
2925     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++)
2926         av_buffer_unref(&s->ps.pps_list[i]);
2927
2928     ff_h2645_packet_uninit(&s->pkt);
2929
2930     return 0;
2931 }
2932
2933 static av_cold int hevc_init_context(AVCodecContext *avctx)
2934 {
2935     HEVCContext *s = avctx->priv_data;
2936     int i;
2937
2938     s->avctx = avctx;
2939
2940     s->tmp_frame = av_frame_alloc();
2941     if (!s->tmp_frame)
2942         goto fail;
2943
2944     s->output_frame = av_frame_alloc();
2945     if (!s->output_frame)
2946         goto fail;
2947
2948     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2949         s->DPB[i].frame = av_frame_alloc();
2950         if (!s->DPB[i].frame)
2951             goto fail;
2952         s->DPB[i].tf.f = s->DPB[i].frame;
2953     }
2954
2955     s->max_ra = INT_MAX;
2956
2957     s->md5_ctx = av_md5_alloc();
2958     if (!s->md5_ctx)
2959         goto fail;
2960
2961     ff_bswapdsp_init(&s->bdsp);
2962
2963     s->context_initialized = 1;
2964
2965     return 0;
2966
2967 fail:
2968     hevc_decode_free(avctx);
2969     return AVERROR(ENOMEM);
2970 }
2971
2972 static int hevc_update_thread_context(AVCodecContext *dst,
2973                                       const AVCodecContext *src)
2974 {
2975     HEVCContext *s  = dst->priv_data;
2976     HEVCContext *s0 = src->priv_data;
2977     int i, ret;
2978
2979     if (!s->context_initialized) {
2980         ret = hevc_init_context(dst);
2981         if (ret < 0)
2982             return ret;
2983     }
2984
2985     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2986         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2987         if (s0->DPB[i].frame->buf[0]) {
2988             ret = hevc_ref_frame(s, &s->DPB[i], &s0->DPB[i]);
2989             if (ret < 0)
2990                 return ret;
2991         }
2992     }
2993
2994     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++) {
2995         av_buffer_unref(&s->ps.vps_list[i]);
2996         if (s0->ps.vps_list[i]) {
2997             s->ps.vps_list[i] = av_buffer_ref(s0->ps.vps_list[i]);
2998             if (!s->ps.vps_list[i])
2999                 return AVERROR(ENOMEM);
3000         }
3001     }
3002
3003     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
3004         av_buffer_unref(&s->ps.sps_list[i]);
3005         if (s0->ps.sps_list[i]) {
3006             s->ps.sps_list[i] = av_buffer_ref(s0->ps.sps_list[i]);
3007             if (!s->ps.sps_list[i])
3008                 return AVERROR(ENOMEM);
3009         }
3010     }
3011
3012     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++) {
3013         av_buffer_unref(&s->ps.pps_list[i]);
3014         if (s0->ps.pps_list[i]) {
3015             s->ps.pps_list[i] = av_buffer_ref(s0->ps.pps_list[i]);
3016             if (!s->ps.pps_list[i])
3017                 return AVERROR(ENOMEM);
3018         }
3019     }
3020
3021     if (s->ps.sps != s0->ps.sps)
3022         ret = set_sps(s, s0->ps.sps, src->pix_fmt);
3023
3024     s->seq_decode = s0->seq_decode;
3025     s->seq_output = s0->seq_output;
3026     s->pocTid0    = s0->pocTid0;
3027     s->max_ra     = s0->max_ra;
3028
3029     s->is_nalff        = s0->is_nalff;
3030     s->nal_length_size = s0->nal_length_size;
3031
3032     if (s0->eos) {
3033         s->seq_decode = (s->seq_decode + 1) & 0xff;
3034         s->max_ra = INT_MAX;
3035     }
3036
3037     return 0;
3038 }
3039
3040 static av_cold int hevc_decode_init(AVCodecContext *avctx)
3041 {
3042     HEVCContext *s = avctx->priv_data;
3043     int ret;
3044
3045     avctx->internal->allocate_progress = 1;
3046
3047     ret = hevc_init_context(avctx);
3048     if (ret < 0)
3049         return ret;
3050
3051     if (avctx->extradata_size > 0 && avctx->extradata) {
3052         ret = hevc_decode_extradata(s, avctx->extradata, avctx->extradata_size);
3053         if (ret < 0) {
3054             hevc_decode_free(avctx);
3055             return ret;
3056         }
3057     }
3058
3059     return 0;
3060 }
3061
3062 static av_cold int hevc_init_thread_copy(AVCodecContext *avctx)
3063 {
3064     HEVCContext *s = avctx->priv_data;
3065     int ret;
3066
3067     memset(s, 0, sizeof(*s));
3068
3069     ret = hevc_init_context(avctx);
3070     if (ret < 0)
3071         return ret;
3072
3073     return 0;
3074 }
3075
3076 static void hevc_decode_flush(AVCodecContext *avctx)
3077 {
3078     HEVCContext *s = avctx->priv_data;
3079     ff_hevc_flush_dpb(s);
3080     s->max_ra = INT_MAX;
3081 }
3082
3083 #define OFFSET(x) offsetof(HEVCContext, x)
3084 #define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
3085
3086 static const AVOption options[] = {
3087     { "apply_defdispwin", "Apply default display window from VUI", OFFSET(apply_defdispwin),
3088         AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, PAR },
3089     { NULL },
3090 };
3091
3092 static const AVClass hevc_decoder_class = {
3093     .class_name = "HEVC decoder",
3094     .item_name  = av_default_item_name,
3095     .option     = options,
3096     .version    = LIBAVUTIL_VERSION_INT,
3097 };
3098
3099 AVCodec ff_hevc_decoder = {
3100     .name                  = "hevc",
3101     .long_name             = NULL_IF_CONFIG_SMALL("HEVC (High Efficiency Video Coding)"),
3102     .type                  = AVMEDIA_TYPE_VIDEO,
3103     .id                    = AV_CODEC_ID_HEVC,
3104     .priv_data_size        = sizeof(HEVCContext),
3105     .priv_class            = &hevc_decoder_class,
3106     .init                  = hevc_decode_init,
3107     .close                 = hevc_decode_free,
3108     .decode                = hevc_decode_frame,
3109     .flush                 = hevc_decode_flush,
3110     .update_thread_context = hevc_update_thread_context,
3111     .init_thread_copy      = hevc_init_thread_copy,
3112     .capabilities          = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY |
3113                              AV_CODEC_CAP_FRAME_THREADS,
3114     .profiles              = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
3115     .caps_internal         = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_THREADSAFE,
3116 };