hevc: reindent after previous commit
[ffmpeg.git] / libavcodec / hevc.c
1 /*
2  * HEVC video decoder
3  *
4  * Copyright (C) 2012 - 2013 Guillaume Martres
5  * Copyright (C) 2012 - 2013 Mickael Raulet
6  * Copyright (C) 2012 - 2013 Gildas Cocherel
7  * Copyright (C) 2012 - 2013 Wassim Hamidouche
8  *
9  * This file is part of Libav.
10  *
11  * Libav is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU Lesser General Public
13  * License as published by the Free Software Foundation; either
14  * version 2.1 of the License, or (at your option) any later version.
15  *
16  * Libav is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19  * Lesser General Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser General Public
22  * License along with Libav; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24  */
25
26 #include "libavutil/attributes.h"
27 #include "libavutil/common.h"
28 #include "libavutil/display.h"
29 #include "libavutil/internal.h"
30 #include "libavutil/md5.h"
31 #include "libavutil/opt.h"
32 #include "libavutil/pixdesc.h"
33 #include "libavutil/stereo3d.h"
34
35 #include "bswapdsp.h"
36 #include "bytestream.h"
37 #include "cabac_functions.h"
38 #include "golomb.h"
39 #include "hevc.h"
40
41 const uint8_t ff_hevc_qpel_extra_before[4] = { 0, 3, 3, 2 };
42 const uint8_t ff_hevc_qpel_extra_after[4]  = { 0, 3, 4, 4 };
43 const uint8_t ff_hevc_qpel_extra[4]        = { 0, 6, 7, 6 };
44
45 static const uint8_t scan_1x1[1] = { 0 };
46
47 static const uint8_t horiz_scan2x2_x[4] = { 0, 1, 0, 1 };
48
49 static const uint8_t horiz_scan2x2_y[4] = { 0, 0, 1, 1 };
50
51 static const uint8_t horiz_scan4x4_x[16] = {
52     0, 1, 2, 3,
53     0, 1, 2, 3,
54     0, 1, 2, 3,
55     0, 1, 2, 3,
56 };
57
58 static const uint8_t horiz_scan4x4_y[16] = {
59     0, 0, 0, 0,
60     1, 1, 1, 1,
61     2, 2, 2, 2,
62     3, 3, 3, 3,
63 };
64
65 static const uint8_t horiz_scan8x8_inv[8][8] = {
66     {  0,  1,  2,  3, 16, 17, 18, 19, },
67     {  4,  5,  6,  7, 20, 21, 22, 23, },
68     {  8,  9, 10, 11, 24, 25, 26, 27, },
69     { 12, 13, 14, 15, 28, 29, 30, 31, },
70     { 32, 33, 34, 35, 48, 49, 50, 51, },
71     { 36, 37, 38, 39, 52, 53, 54, 55, },
72     { 40, 41, 42, 43, 56, 57, 58, 59, },
73     { 44, 45, 46, 47, 60, 61, 62, 63, },
74 };
75
76 static const uint8_t diag_scan2x2_x[4] = { 0, 0, 1, 1 };
77
78 static const uint8_t diag_scan2x2_y[4] = { 0, 1, 0, 1 };
79
80 static const uint8_t diag_scan2x2_inv[2][2] = {
81     { 0, 2, },
82     { 1, 3, },
83 };
84
85 const uint8_t ff_hevc_diag_scan4x4_x[16] = {
86     0, 0, 1, 0,
87     1, 2, 0, 1,
88     2, 3, 1, 2,
89     3, 2, 3, 3,
90 };
91
92 const uint8_t ff_hevc_diag_scan4x4_y[16] = {
93     0, 1, 0, 2,
94     1, 0, 3, 2,
95     1, 0, 3, 2,
96     1, 3, 2, 3,
97 };
98
99 static const uint8_t diag_scan4x4_inv[4][4] = {
100     { 0,  2,  5,  9, },
101     { 1,  4,  8, 12, },
102     { 3,  7, 11, 14, },
103     { 6, 10, 13, 15, },
104 };
105
106 const uint8_t ff_hevc_diag_scan8x8_x[64] = {
107     0, 0, 1, 0,
108     1, 2, 0, 1,
109     2, 3, 0, 1,
110     2, 3, 4, 0,
111     1, 2, 3, 4,
112     5, 0, 1, 2,
113     3, 4, 5, 6,
114     0, 1, 2, 3,
115     4, 5, 6, 7,
116     1, 2, 3, 4,
117     5, 6, 7, 2,
118     3, 4, 5, 6,
119     7, 3, 4, 5,
120     6, 7, 4, 5,
121     6, 7, 5, 6,
122     7, 6, 7, 7,
123 };
124
125 const uint8_t ff_hevc_diag_scan8x8_y[64] = {
126     0, 1, 0, 2,
127     1, 0, 3, 2,
128     1, 0, 4, 3,
129     2, 1, 0, 5,
130     4, 3, 2, 1,
131     0, 6, 5, 4,
132     3, 2, 1, 0,
133     7, 6, 5, 4,
134     3, 2, 1, 0,
135     7, 6, 5, 4,
136     3, 2, 1, 7,
137     6, 5, 4, 3,
138     2, 7, 6, 5,
139     4, 3, 7, 6,
140     5, 4, 7, 6,
141     5, 7, 6, 7,
142 };
143
144 static const uint8_t diag_scan8x8_inv[8][8] = {
145     {  0,  2,  5,  9, 14, 20, 27, 35, },
146     {  1,  4,  8, 13, 19, 26, 34, 42, },
147     {  3,  7, 12, 18, 25, 33, 41, 48, },
148     {  6, 11, 17, 24, 32, 40, 47, 53, },
149     { 10, 16, 23, 31, 39, 46, 52, 57, },
150     { 15, 22, 30, 38, 45, 51, 56, 60, },
151     { 21, 29, 37, 44, 50, 55, 59, 62, },
152     { 28, 36, 43, 49, 54, 58, 61, 63, },
153 };
154
155 /**
156  * NOTE: Each function hls_foo correspond to the function foo in the
157  * specification (HLS stands for High Level Syntax).
158  */
159
160 /**
161  * Section 5.7
162  */
163
164 /* free everything allocated  by pic_arrays_init() */
165 static void pic_arrays_free(HEVCContext *s)
166 {
167     av_freep(&s->sao);
168     av_freep(&s->deblock);
169
170     av_freep(&s->skip_flag);
171     av_freep(&s->tab_ct_depth);
172
173     av_freep(&s->tab_ipm);
174     av_freep(&s->cbf_luma);
175     av_freep(&s->is_pcm);
176
177     av_freep(&s->qp_y_tab);
178     av_freep(&s->tab_slice_address);
179     av_freep(&s->filter_slice_edges);
180
181     av_freep(&s->horizontal_bs);
182     av_freep(&s->vertical_bs);
183
184     av_buffer_pool_uninit(&s->tab_mvf_pool);
185     av_buffer_pool_uninit(&s->rpl_tab_pool);
186 }
187
188 /* allocate arrays that depend on frame dimensions */
189 static int pic_arrays_init(HEVCContext *s, const HEVCSPS *sps)
190 {
191     int log2_min_cb_size = sps->log2_min_cb_size;
192     int width            = sps->width;
193     int height           = sps->height;
194     int pic_size_in_ctb  = ((width  >> log2_min_cb_size) + 1) *
195                            ((height >> log2_min_cb_size) + 1);
196     int ctb_count        = sps->ctb_width * sps->ctb_height;
197     int min_pu_size      = sps->min_pu_width * sps->min_pu_height;
198
199     s->bs_width  = width  >> 3;
200     s->bs_height = height >> 3;
201
202     s->sao           = av_mallocz_array(ctb_count, sizeof(*s->sao));
203     s->deblock       = av_mallocz_array(ctb_count, sizeof(*s->deblock));
204     if (!s->sao || !s->deblock)
205         goto fail;
206
207     s->skip_flag    = av_malloc(pic_size_in_ctb);
208     s->tab_ct_depth = av_malloc(sps->min_cb_height * sps->min_cb_width);
209     if (!s->skip_flag || !s->tab_ct_depth)
210         goto fail;
211
212     s->cbf_luma = av_malloc(sps->min_tb_width * sps->min_tb_height);
213     s->tab_ipm  = av_mallocz(min_pu_size);
214     s->is_pcm   = av_malloc(min_pu_size);
215     if (!s->tab_ipm || !s->cbf_luma || !s->is_pcm)
216         goto fail;
217
218     s->filter_slice_edges = av_malloc(ctb_count);
219     s->tab_slice_address  = av_malloc(pic_size_in_ctb *
220                                       sizeof(*s->tab_slice_address));
221     s->qp_y_tab           = av_malloc(pic_size_in_ctb *
222                                       sizeof(*s->qp_y_tab));
223     if (!s->qp_y_tab || !s->filter_slice_edges || !s->tab_slice_address)
224         goto fail;
225
226     s->horizontal_bs = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
227     s->vertical_bs   = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
228     if (!s->horizontal_bs || !s->vertical_bs)
229         goto fail;
230
231     s->tab_mvf_pool = av_buffer_pool_init(min_pu_size * sizeof(MvField),
232                                           av_buffer_alloc);
233     s->rpl_tab_pool = av_buffer_pool_init(ctb_count * sizeof(RefPicListTab),
234                                           av_buffer_allocz);
235     if (!s->tab_mvf_pool || !s->rpl_tab_pool)
236         goto fail;
237
238     return 0;
239
240 fail:
241     pic_arrays_free(s);
242     return AVERROR(ENOMEM);
243 }
244
245 static void pred_weight_table(HEVCContext *s, GetBitContext *gb)
246 {
247     int i = 0;
248     int j = 0;
249     uint8_t luma_weight_l0_flag[16];
250     uint8_t chroma_weight_l0_flag[16];
251     uint8_t luma_weight_l1_flag[16];
252     uint8_t chroma_weight_l1_flag[16];
253
254     s->sh.luma_log2_weight_denom = av_clip_c(get_ue_golomb_long(gb), 0, 7);
255     if (s->sps->chroma_format_idc != 0) {
256         int delta = get_se_golomb(gb);
257         s->sh.chroma_log2_weight_denom = av_clip_c(s->sh.luma_log2_weight_denom + delta, 0, 7);
258     }
259
260     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
261         luma_weight_l0_flag[i] = get_bits1(gb);
262         if (!luma_weight_l0_flag[i]) {
263             s->sh.luma_weight_l0[i] = 1 << s->sh.luma_log2_weight_denom;
264             s->sh.luma_offset_l0[i] = 0;
265         }
266     }
267     if (s->sps->chroma_format_idc != 0) { // FIXME: invert "if" and "for"
268         for (i = 0; i < s->sh.nb_refs[L0]; i++)
269             chroma_weight_l0_flag[i] = get_bits1(gb);
270     } else {
271         for (i = 0; i < s->sh.nb_refs[L0]; i++)
272             chroma_weight_l0_flag[i] = 0;
273     }
274     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
275         if (luma_weight_l0_flag[i]) {
276             int delta_luma_weight_l0 = get_se_golomb(gb);
277             s->sh.luma_weight_l0[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l0;
278             s->sh.luma_offset_l0[i] = get_se_golomb(gb);
279         }
280         if (chroma_weight_l0_flag[i]) {
281             for (j = 0; j < 2; j++) {
282                 int delta_chroma_weight_l0 = get_se_golomb(gb);
283                 int delta_chroma_offset_l0 = get_se_golomb(gb);
284                 s->sh.chroma_weight_l0[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l0;
285                 s->sh.chroma_offset_l0[i][j] = av_clip_c((delta_chroma_offset_l0 - ((128 * s->sh.chroma_weight_l0[i][j])
286                                                                                     >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
287             }
288         } else {
289             s->sh.chroma_weight_l0[i][0] = 1 << s->sh.chroma_log2_weight_denom;
290             s->sh.chroma_offset_l0[i][0] = 0;
291             s->sh.chroma_weight_l0[i][1] = 1 << s->sh.chroma_log2_weight_denom;
292             s->sh.chroma_offset_l0[i][1] = 0;
293         }
294     }
295     if (s->sh.slice_type == B_SLICE) {
296         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
297             luma_weight_l1_flag[i] = get_bits1(gb);
298             if (!luma_weight_l1_flag[i]) {
299                 s->sh.luma_weight_l1[i] = 1 << s->sh.luma_log2_weight_denom;
300                 s->sh.luma_offset_l1[i] = 0;
301             }
302         }
303         if (s->sps->chroma_format_idc != 0) {
304             for (i = 0; i < s->sh.nb_refs[L1]; i++)
305                 chroma_weight_l1_flag[i] = get_bits1(gb);
306         } else {
307             for (i = 0; i < s->sh.nb_refs[L1]; i++)
308                 chroma_weight_l1_flag[i] = 0;
309         }
310         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
311             if (luma_weight_l1_flag[i]) {
312                 int delta_luma_weight_l1 = get_se_golomb(gb);
313                 s->sh.luma_weight_l1[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l1;
314                 s->sh.luma_offset_l1[i] = get_se_golomb(gb);
315             }
316             if (chroma_weight_l1_flag[i]) {
317                 for (j = 0; j < 2; j++) {
318                     int delta_chroma_weight_l1 = get_se_golomb(gb);
319                     int delta_chroma_offset_l1 = get_se_golomb(gb);
320                     s->sh.chroma_weight_l1[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l1;
321                     s->sh.chroma_offset_l1[i][j] = av_clip_c((delta_chroma_offset_l1 - ((128 * s->sh.chroma_weight_l1[i][j])
322                                                                                         >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
323                 }
324             } else {
325                 s->sh.chroma_weight_l1[i][0] = 1 << s->sh.chroma_log2_weight_denom;
326                 s->sh.chroma_offset_l1[i][0] = 0;
327                 s->sh.chroma_weight_l1[i][1] = 1 << s->sh.chroma_log2_weight_denom;
328                 s->sh.chroma_offset_l1[i][1] = 0;
329             }
330         }
331     }
332 }
333
334 static int decode_lt_rps(HEVCContext *s, LongTermRPS *rps, GetBitContext *gb)
335 {
336     const HEVCSPS *sps = s->sps;
337     int max_poc_lsb    = 1 << sps->log2_max_poc_lsb;
338     int prev_delta_msb = 0;
339     unsigned int nb_sps = 0, nb_sh;
340     int i;
341
342     rps->nb_refs = 0;
343     if (!sps->long_term_ref_pics_present_flag)
344         return 0;
345
346     if (sps->num_long_term_ref_pics_sps > 0)
347         nb_sps = get_ue_golomb_long(gb);
348     nb_sh = get_ue_golomb_long(gb);
349
350     if (nb_sh + nb_sps > FF_ARRAY_ELEMS(rps->poc))
351         return AVERROR_INVALIDDATA;
352
353     rps->nb_refs = nb_sh + nb_sps;
354
355     for (i = 0; i < rps->nb_refs; i++) {
356         uint8_t delta_poc_msb_present;
357
358         if (i < nb_sps) {
359             uint8_t lt_idx_sps = 0;
360
361             if (sps->num_long_term_ref_pics_sps > 1)
362                 lt_idx_sps = get_bits(gb, av_ceil_log2(sps->num_long_term_ref_pics_sps));
363
364             rps->poc[i]  = sps->lt_ref_pic_poc_lsb_sps[lt_idx_sps];
365             rps->used[i] = sps->used_by_curr_pic_lt_sps_flag[lt_idx_sps];
366         } else {
367             rps->poc[i]  = get_bits(gb, sps->log2_max_poc_lsb);
368             rps->used[i] = get_bits1(gb);
369         }
370
371         delta_poc_msb_present = get_bits1(gb);
372         if (delta_poc_msb_present) {
373             int delta = get_ue_golomb_long(gb);
374
375             if (i && i != nb_sps)
376                 delta += prev_delta_msb;
377
378             rps->poc[i] += s->poc - delta * max_poc_lsb - s->sh.pic_order_cnt_lsb;
379             prev_delta_msb = delta;
380         }
381     }
382
383     return 0;
384 }
385
386 static int set_sps(HEVCContext *s, const HEVCSPS *sps)
387 {
388     #define HWACCEL_MAX (0)
389     enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmt = pix_fmts;
390     int ret;
391     unsigned int num = 0, den = 0;
392
393     pic_arrays_free(s);
394     ret = pic_arrays_init(s, sps);
395     if (ret < 0)
396         goto fail;
397
398     s->avctx->coded_width         = sps->width;
399     s->avctx->coded_height        = sps->height;
400     s->avctx->width               = sps->output_width;
401     s->avctx->height              = sps->output_height;
402     s->avctx->has_b_frames        = sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics;
403
404     *fmt++ = sps->pix_fmt;
405     *fmt = AV_PIX_FMT_NONE;
406
407     ret = ff_get_format(s->avctx, pix_fmts);
408     if (ret < 0)
409         goto fail;
410     s->avctx->pix_fmt = ret;
411
412     ff_set_sar(s->avctx, sps->vui.sar);
413
414     if (sps->vui.video_signal_type_present_flag)
415         s->avctx->color_range = sps->vui.video_full_range_flag ? AVCOL_RANGE_JPEG
416                                                                : AVCOL_RANGE_MPEG;
417     else
418         s->avctx->color_range = AVCOL_RANGE_MPEG;
419
420     if (sps->vui.colour_description_present_flag) {
421         s->avctx->color_primaries = sps->vui.colour_primaries;
422         s->avctx->color_trc       = sps->vui.transfer_characteristic;
423         s->avctx->colorspace      = sps->vui.matrix_coeffs;
424     } else {
425         s->avctx->color_primaries = AVCOL_PRI_UNSPECIFIED;
426         s->avctx->color_trc       = AVCOL_TRC_UNSPECIFIED;
427         s->avctx->colorspace      = AVCOL_SPC_UNSPECIFIED;
428     }
429
430     ff_hevc_pred_init(&s->hpc,     sps->bit_depth);
431     ff_hevc_dsp_init (&s->hevcdsp, sps->bit_depth);
432     ff_videodsp_init (&s->vdsp,    sps->bit_depth);
433
434     if (sps->sao_enabled && !s->avctx->hwaccel) {
435         av_frame_unref(s->tmp_frame);
436         ret = ff_get_buffer(s->avctx, s->tmp_frame, AV_GET_BUFFER_FLAG_REF);
437         if (ret < 0)
438             goto fail;
439         s->frame = s->tmp_frame;
440     }
441
442     s->sps = sps;
443     s->vps = (HEVCVPS*) s->vps_list[s->sps->vps_id]->data;
444
445     if (s->vps->vps_timing_info_present_flag) {
446         num = s->vps->vps_num_units_in_tick;
447         den = s->vps->vps_time_scale;
448     } else if (sps->vui.vui_timing_info_present_flag) {
449         num = sps->vui.vui_num_units_in_tick;
450         den = sps->vui.vui_time_scale;
451     }
452
453     if (num != 0 && den != 0)
454         av_reduce(&s->avctx->framerate.den, &s->avctx->framerate.num,
455                   num, den, 1 << 30);
456
457     return 0;
458
459 fail:
460     pic_arrays_free(s);
461     s->sps = NULL;
462     return ret;
463 }
464
465 static int hls_slice_header(HEVCContext *s)
466 {
467     GetBitContext *gb = &s->HEVClc.gb;
468     SliceHeader *sh   = &s->sh;
469     int i, ret;
470
471     // Coded parameters
472     sh->first_slice_in_pic_flag = get_bits1(gb);
473     if ((IS_IDR(s) || IS_BLA(s)) && sh->first_slice_in_pic_flag) {
474         s->seq_decode = (s->seq_decode + 1) & 0xff;
475         s->max_ra     = INT_MAX;
476         if (IS_IDR(s))
477             ff_hevc_clear_refs(s);
478     }
479     if (IS_IRAP(s))
480         sh->no_output_of_prior_pics_flag = get_bits1(gb);
481
482     sh->pps_id = get_ue_golomb_long(gb);
483     if (sh->pps_id >= MAX_PPS_COUNT || !s->pps_list[sh->pps_id]) {
484         av_log(s->avctx, AV_LOG_ERROR, "PPS id out of range: %d\n", sh->pps_id);
485         return AVERROR_INVALIDDATA;
486     }
487     if (!sh->first_slice_in_pic_flag &&
488         s->pps != (HEVCPPS*)s->pps_list[sh->pps_id]->data) {
489         av_log(s->avctx, AV_LOG_ERROR, "PPS changed between slices.\n");
490         return AVERROR_INVALIDDATA;
491     }
492     s->pps = (HEVCPPS*)s->pps_list[sh->pps_id]->data;
493
494     if (s->sps != (HEVCSPS*)s->sps_list[s->pps->sps_id]->data) {
495         s->sps = (HEVCSPS*)s->sps_list[s->pps->sps_id]->data;
496
497         ff_hevc_clear_refs(s);
498         ret = set_sps(s, s->sps);
499         if (ret < 0)
500             return ret;
501
502         s->seq_decode = (s->seq_decode + 1) & 0xff;
503         s->max_ra     = INT_MAX;
504     }
505
506     s->avctx->profile = s->sps->ptl.general_ptl.profile_idc;
507     s->avctx->level   = s->sps->ptl.general_ptl.level_idc;
508
509     sh->dependent_slice_segment_flag = 0;
510     if (!sh->first_slice_in_pic_flag) {
511         int slice_address_length;
512
513         if (s->pps->dependent_slice_segments_enabled_flag)
514             sh->dependent_slice_segment_flag = get_bits1(gb);
515
516         slice_address_length = av_ceil_log2(s->sps->ctb_width *
517                                             s->sps->ctb_height);
518         sh->slice_segment_addr = get_bits(gb, slice_address_length);
519         if (sh->slice_segment_addr >= s->sps->ctb_width * s->sps->ctb_height) {
520             av_log(s->avctx, AV_LOG_ERROR,
521                    "Invalid slice segment address: %u.\n",
522                    sh->slice_segment_addr);
523             return AVERROR_INVALIDDATA;
524         }
525
526         if (!sh->dependent_slice_segment_flag) {
527             sh->slice_addr = sh->slice_segment_addr;
528             s->slice_idx++;
529         }
530     } else {
531         sh->slice_segment_addr = sh->slice_addr = 0;
532         s->slice_idx           = 0;
533         s->slice_initialized   = 0;
534     }
535
536     if (!sh->dependent_slice_segment_flag) {
537         s->slice_initialized = 0;
538
539         for (i = 0; i < s->pps->num_extra_slice_header_bits; i++)
540             skip_bits(gb, 1);  // slice_reserved_undetermined_flag[]
541
542         sh->slice_type = get_ue_golomb_long(gb);
543         if (!(sh->slice_type == I_SLICE ||
544               sh->slice_type == P_SLICE ||
545               sh->slice_type == B_SLICE)) {
546             av_log(s->avctx, AV_LOG_ERROR, "Unknown slice type: %d.\n",
547                    sh->slice_type);
548             return AVERROR_INVALIDDATA;
549         }
550         if (IS_IRAP(s) && sh->slice_type != I_SLICE) {
551             av_log(s->avctx, AV_LOG_ERROR, "Inter slices in an IRAP frame.\n");
552             return AVERROR_INVALIDDATA;
553         }
554
555         // when flag is not present, picture is inferred to be output
556         sh->pic_output_flag = 1;
557         if (s->pps->output_flag_present_flag)
558             sh->pic_output_flag = get_bits1(gb);
559
560         if (s->sps->separate_colour_plane_flag)
561             sh->colour_plane_id = get_bits(gb, 2);
562
563         if (!IS_IDR(s)) {
564             int poc;
565
566             sh->pic_order_cnt_lsb = get_bits(gb, s->sps->log2_max_poc_lsb);
567             poc = ff_hevc_compute_poc(s, sh->pic_order_cnt_lsb);
568             if (!sh->first_slice_in_pic_flag && poc != s->poc) {
569                 av_log(s->avctx, AV_LOG_WARNING,
570                        "Ignoring POC change between slices: %d -> %d\n", s->poc, poc);
571                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
572                     return AVERROR_INVALIDDATA;
573                 poc = s->poc;
574             }
575             s->poc = poc;
576
577             sh->short_term_ref_pic_set_sps_flag = get_bits1(gb);
578             if (!sh->short_term_ref_pic_set_sps_flag) {
579                 int pos = get_bits_left(gb);
580                 ret = ff_hevc_decode_short_term_rps(s, &sh->slice_rps, s->sps, 1);
581                 if (ret < 0)
582                     return ret;
583
584                 sh->short_term_ref_pic_set_size = pos - get_bits_left(gb);
585                 sh->short_term_rps = &sh->slice_rps;
586             } else {
587                 int numbits, rps_idx;
588
589                 if (!s->sps->nb_st_rps) {
590                     av_log(s->avctx, AV_LOG_ERROR, "No ref lists in the SPS.\n");
591                     return AVERROR_INVALIDDATA;
592                 }
593
594                 numbits = av_ceil_log2(s->sps->nb_st_rps);
595                 rps_idx = numbits > 0 ? get_bits(gb, numbits) : 0;
596                 sh->short_term_rps = &s->sps->st_rps[rps_idx];
597             }
598
599             ret = decode_lt_rps(s, &sh->long_term_rps, gb);
600             if (ret < 0) {
601                 av_log(s->avctx, AV_LOG_WARNING, "Invalid long term RPS.\n");
602                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
603                     return AVERROR_INVALIDDATA;
604             }
605
606             if (s->sps->sps_temporal_mvp_enabled_flag)
607                 sh->slice_temporal_mvp_enabled_flag = get_bits1(gb);
608             else
609                 sh->slice_temporal_mvp_enabled_flag = 0;
610         } else {
611             s->sh.short_term_rps = NULL;
612             s->poc               = 0;
613         }
614
615         /* 8.3.1 */
616         if (s->temporal_id == 0 &&
617             s->nal_unit_type != NAL_TRAIL_N &&
618             s->nal_unit_type != NAL_TSA_N   &&
619             s->nal_unit_type != NAL_STSA_N  &&
620             s->nal_unit_type != NAL_RADL_N  &&
621             s->nal_unit_type != NAL_RADL_R  &&
622             s->nal_unit_type != NAL_RASL_N  &&
623             s->nal_unit_type != NAL_RASL_R)
624             s->pocTid0 = s->poc;
625
626         if (s->sps->sao_enabled) {
627             sh->slice_sample_adaptive_offset_flag[0] = get_bits1(gb);
628             sh->slice_sample_adaptive_offset_flag[1] =
629             sh->slice_sample_adaptive_offset_flag[2] = get_bits1(gb);
630         } else {
631             sh->slice_sample_adaptive_offset_flag[0] = 0;
632             sh->slice_sample_adaptive_offset_flag[1] = 0;
633             sh->slice_sample_adaptive_offset_flag[2] = 0;
634         }
635
636         sh->nb_refs[L0] = sh->nb_refs[L1] = 0;
637         if (sh->slice_type == P_SLICE || sh->slice_type == B_SLICE) {
638             int nb_refs;
639
640             sh->nb_refs[L0] = s->pps->num_ref_idx_l0_default_active;
641             if (sh->slice_type == B_SLICE)
642                 sh->nb_refs[L1] = s->pps->num_ref_idx_l1_default_active;
643
644             if (get_bits1(gb)) { // num_ref_idx_active_override_flag
645                 sh->nb_refs[L0] = get_ue_golomb_long(gb) + 1;
646                 if (sh->slice_type == B_SLICE)
647                     sh->nb_refs[L1] = get_ue_golomb_long(gb) + 1;
648             }
649             if (sh->nb_refs[L0] > MAX_REFS || sh->nb_refs[L1] > MAX_REFS) {
650                 av_log(s->avctx, AV_LOG_ERROR, "Too many refs: %d/%d.\n",
651                        sh->nb_refs[L0], sh->nb_refs[L1]);
652                 return AVERROR_INVALIDDATA;
653             }
654
655             sh->rpl_modification_flag[0] = 0;
656             sh->rpl_modification_flag[1] = 0;
657             nb_refs = ff_hevc_frame_nb_refs(s);
658             if (!nb_refs) {
659                 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
660                 return AVERROR_INVALIDDATA;
661             }
662
663             if (s->pps->lists_modification_present_flag && nb_refs > 1) {
664                 sh->rpl_modification_flag[0] = get_bits1(gb);
665                 if (sh->rpl_modification_flag[0]) {
666                     for (i = 0; i < sh->nb_refs[L0]; i++)
667                         sh->list_entry_lx[0][i] = get_bits(gb, av_ceil_log2(nb_refs));
668                 }
669
670                 if (sh->slice_type == B_SLICE) {
671                     sh->rpl_modification_flag[1] = get_bits1(gb);
672                     if (sh->rpl_modification_flag[1] == 1)
673                         for (i = 0; i < sh->nb_refs[L1]; i++)
674                             sh->list_entry_lx[1][i] = get_bits(gb, av_ceil_log2(nb_refs));
675                 }
676             }
677
678             if (sh->slice_type == B_SLICE)
679                 sh->mvd_l1_zero_flag = get_bits1(gb);
680
681             if (s->pps->cabac_init_present_flag)
682                 sh->cabac_init_flag = get_bits1(gb);
683             else
684                 sh->cabac_init_flag = 0;
685
686             sh->collocated_ref_idx = 0;
687             if (sh->slice_temporal_mvp_enabled_flag) {
688                 sh->collocated_list = L0;
689                 if (sh->slice_type == B_SLICE)
690                     sh->collocated_list = !get_bits1(gb);
691
692                 if (sh->nb_refs[sh->collocated_list] > 1) {
693                     sh->collocated_ref_idx = get_ue_golomb_long(gb);
694                     if (sh->collocated_ref_idx >= sh->nb_refs[sh->collocated_list]) {
695                         av_log(s->avctx, AV_LOG_ERROR,
696                                "Invalid collocated_ref_idx: %d.\n",
697                                sh->collocated_ref_idx);
698                         return AVERROR_INVALIDDATA;
699                     }
700                 }
701             }
702
703             if ((s->pps->weighted_pred_flag   && sh->slice_type == P_SLICE) ||
704                 (s->pps->weighted_bipred_flag && sh->slice_type == B_SLICE)) {
705                 pred_weight_table(s, gb);
706             }
707
708             sh->max_num_merge_cand = 5 - get_ue_golomb_long(gb);
709             if (sh->max_num_merge_cand < 1 || sh->max_num_merge_cand > 5) {
710                 av_log(s->avctx, AV_LOG_ERROR,
711                        "Invalid number of merging MVP candidates: %d.\n",
712                        sh->max_num_merge_cand);
713                 return AVERROR_INVALIDDATA;
714             }
715         }
716
717         sh->slice_qp_delta = get_se_golomb(gb);
718
719         if (s->pps->pic_slice_level_chroma_qp_offsets_present_flag) {
720             sh->slice_cb_qp_offset = get_se_golomb(gb);
721             sh->slice_cr_qp_offset = get_se_golomb(gb);
722         } else {
723             sh->slice_cb_qp_offset = 0;
724             sh->slice_cr_qp_offset = 0;
725         }
726
727         if (s->pps->deblocking_filter_control_present_flag) {
728             int deblocking_filter_override_flag = 0;
729
730             if (s->pps->deblocking_filter_override_enabled_flag)
731                 deblocking_filter_override_flag = get_bits1(gb);
732
733             if (deblocking_filter_override_flag) {
734                 sh->disable_deblocking_filter_flag = get_bits1(gb);
735                 if (!sh->disable_deblocking_filter_flag) {
736                     sh->beta_offset = get_se_golomb(gb) * 2;
737                     sh->tc_offset   = get_se_golomb(gb) * 2;
738                 }
739             } else {
740                 sh->disable_deblocking_filter_flag = s->pps->disable_dbf;
741                 sh->beta_offset                    = s->pps->beta_offset;
742                 sh->tc_offset                      = s->pps->tc_offset;
743             }
744         } else {
745             sh->disable_deblocking_filter_flag = 0;
746             sh->beta_offset                    = 0;
747             sh->tc_offset                      = 0;
748         }
749
750         if (s->pps->seq_loop_filter_across_slices_enabled_flag &&
751             (sh->slice_sample_adaptive_offset_flag[0] ||
752              sh->slice_sample_adaptive_offset_flag[1] ||
753              !sh->disable_deblocking_filter_flag)) {
754             sh->slice_loop_filter_across_slices_enabled_flag = get_bits1(gb);
755         } else {
756             sh->slice_loop_filter_across_slices_enabled_flag = s->pps->seq_loop_filter_across_slices_enabled_flag;
757         }
758     } else if (!s->slice_initialized) {
759         av_log(s->avctx, AV_LOG_ERROR, "Independent slice segment missing.\n");
760         return AVERROR_INVALIDDATA;
761     }
762
763     sh->num_entry_point_offsets = 0;
764     if (s->pps->tiles_enabled_flag || s->pps->entropy_coding_sync_enabled_flag) {
765         sh->num_entry_point_offsets = get_ue_golomb_long(gb);
766         if (sh->num_entry_point_offsets > 0) {
767             int offset_len = get_ue_golomb_long(gb) + 1;
768
769             for (i = 0; i < sh->num_entry_point_offsets; i++)
770                 skip_bits(gb, offset_len);
771         }
772     }
773
774     if (s->pps->slice_header_extension_present_flag) {
775         unsigned int length = get_ue_golomb_long(gb);
776         for (i = 0; i < length; i++)
777             skip_bits(gb, 8);  // slice_header_extension_data_byte
778     }
779
780     // Inferred parameters
781     sh->slice_qp = 26 + s->pps->pic_init_qp_minus26 + sh->slice_qp_delta;
782     if (sh->slice_qp > 51 ||
783         sh->slice_qp < -s->sps->qp_bd_offset) {
784         av_log(s->avctx, AV_LOG_ERROR,
785                "The slice_qp %d is outside the valid range "
786                "[%d, 51].\n",
787                sh->slice_qp,
788                -s->sps->qp_bd_offset);
789         return AVERROR_INVALIDDATA;
790     }
791
792     sh->slice_ctb_addr_rs = sh->slice_segment_addr;
793
794     if (!s->sh.slice_ctb_addr_rs && s->sh.dependent_slice_segment_flag) {
795         av_log(s->avctx, AV_LOG_ERROR, "Impossible slice segment.\n");
796         return AVERROR_INVALIDDATA;
797     }
798
799     s->HEVClc.first_qp_group = !s->sh.dependent_slice_segment_flag;
800
801     if (!s->pps->cu_qp_delta_enabled_flag)
802         s->HEVClc.qp_y = FFUMOD(s->sh.slice_qp + 52 + 2 * s->sps->qp_bd_offset,
803                                 52 + s->sps->qp_bd_offset) - s->sps->qp_bd_offset;
804
805     s->slice_initialized = 1;
806
807     return 0;
808 }
809
810 #define CTB(tab, x, y) ((tab)[(y) * s->sps->ctb_width + (x)])
811
812 #define SET_SAO(elem, value)                            \
813 do {                                                    \
814     if (!sao_merge_up_flag && !sao_merge_left_flag)     \
815         sao->elem = value;                              \
816     else if (sao_merge_left_flag)                       \
817         sao->elem = CTB(s->sao, rx-1, ry).elem;         \
818     else if (sao_merge_up_flag)                         \
819         sao->elem = CTB(s->sao, rx, ry-1).elem;         \
820     else                                                \
821         sao->elem = 0;                                  \
822 } while (0)
823
824 static void hls_sao_param(HEVCContext *s, int rx, int ry)
825 {
826     HEVCLocalContext *lc    = &s->HEVClc;
827     int sao_merge_left_flag = 0;
828     int sao_merge_up_flag   = 0;
829     int shift               = s->sps->bit_depth - FFMIN(s->sps->bit_depth, 10);
830     SAOParams *sao          = &CTB(s->sao, rx, ry);
831     int c_idx, i;
832
833     if (s->sh.slice_sample_adaptive_offset_flag[0] ||
834         s->sh.slice_sample_adaptive_offset_flag[1]) {
835         if (rx > 0) {
836             if (lc->ctb_left_flag)
837                 sao_merge_left_flag = ff_hevc_sao_merge_flag_decode(s);
838         }
839         if (ry > 0 && !sao_merge_left_flag) {
840             if (lc->ctb_up_flag)
841                 sao_merge_up_flag = ff_hevc_sao_merge_flag_decode(s);
842         }
843     }
844
845     for (c_idx = 0; c_idx < 3; c_idx++) {
846         if (!s->sh.slice_sample_adaptive_offset_flag[c_idx]) {
847             sao->type_idx[c_idx] = SAO_NOT_APPLIED;
848             continue;
849         }
850
851         if (c_idx == 2) {
852             sao->type_idx[2] = sao->type_idx[1];
853             sao->eo_class[2] = sao->eo_class[1];
854         } else {
855             SET_SAO(type_idx[c_idx], ff_hevc_sao_type_idx_decode(s));
856         }
857
858         if (sao->type_idx[c_idx] == SAO_NOT_APPLIED)
859             continue;
860
861         for (i = 0; i < 4; i++)
862             SET_SAO(offset_abs[c_idx][i], ff_hevc_sao_offset_abs_decode(s));
863
864         if (sao->type_idx[c_idx] == SAO_BAND) {
865             for (i = 0; i < 4; i++) {
866                 if (sao->offset_abs[c_idx][i]) {
867                     SET_SAO(offset_sign[c_idx][i],
868                             ff_hevc_sao_offset_sign_decode(s));
869                 } else {
870                     sao->offset_sign[c_idx][i] = 0;
871                 }
872             }
873             SET_SAO(band_position[c_idx], ff_hevc_sao_band_position_decode(s));
874         } else if (c_idx != 2) {
875             SET_SAO(eo_class[c_idx], ff_hevc_sao_eo_class_decode(s));
876         }
877
878         // Inferred parameters
879         sao->offset_val[c_idx][0] = 0;
880         for (i = 0; i < 4; i++) {
881             sao->offset_val[c_idx][i + 1] = sao->offset_abs[c_idx][i] << shift;
882             if (sao->type_idx[c_idx] == SAO_EDGE) {
883                 if (i > 1)
884                     sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
885             } else if (sao->offset_sign[c_idx][i]) {
886                 sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
887             }
888         }
889     }
890 }
891
892 #undef SET_SAO
893 #undef CTB
894
895 static void hls_residual_coding(HEVCContext *s, int x0, int y0,
896                                 int log2_trafo_size, enum ScanType scan_idx,
897                                 int c_idx)
898 {
899 #define GET_COORD(offset, n)                                    \
900     do {                                                        \
901         x_c = (scan_x_cg[offset >> 4] << 2) + scan_x_off[n];    \
902         y_c = (scan_y_cg[offset >> 4] << 2) + scan_y_off[n];    \
903     } while (0)
904     HEVCLocalContext *lc    = &s->HEVClc;
905     int transform_skip_flag = 0;
906
907     int last_significant_coeff_x, last_significant_coeff_y;
908     int last_scan_pos;
909     int n_end;
910     int num_coeff    = 0;
911     int greater1_ctx = 1;
912
913     int num_last_subset;
914     int x_cg_last_sig, y_cg_last_sig;
915
916     const uint8_t *scan_x_cg, *scan_y_cg, *scan_x_off, *scan_y_off;
917
918     ptrdiff_t stride = s->frame->linesize[c_idx];
919     int hshift       = s->sps->hshift[c_idx];
920     int vshift       = s->sps->vshift[c_idx];
921     uint8_t *dst     = &s->frame->data[c_idx][(y0 >> vshift) * stride +
922                                               ((x0 >> hshift) << s->sps->pixel_shift)];
923     DECLARE_ALIGNED(16, int16_t, coeffs[MAX_TB_SIZE * MAX_TB_SIZE]) = { 0 };
924     DECLARE_ALIGNED(8, uint8_t, significant_coeff_group_flag[8][8]) = { { 0 } };
925
926     int trafo_size = 1 << log2_trafo_size;
927     int i, qp, shift, add, scale, scale_m;
928     const uint8_t level_scale[] = { 40, 45, 51, 57, 64, 72 };
929     const uint8_t *scale_matrix;
930     uint8_t dc_scale;
931
932     // Derive QP for dequant
933     if (!lc->cu.cu_transquant_bypass_flag) {
934         static const int qp_c[] = {
935             29, 30, 31, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37
936         };
937
938         static const uint8_t rem6[51 + 2 * 6 + 1] = {
939             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
940             3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
941             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
942         };
943
944         static const uint8_t div6[51 + 2 * 6 + 1] = {
945             0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,  3,  3,  3,
946             3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6,  6,  6,  6,
947             7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10,
948         };
949         int qp_y = lc->qp_y;
950
951         if (c_idx == 0) {
952             qp = qp_y + s->sps->qp_bd_offset;
953         } else {
954             int qp_i, offset;
955
956             if (c_idx == 1)
957                 offset = s->pps->cb_qp_offset + s->sh.slice_cb_qp_offset;
958             else
959                 offset = s->pps->cr_qp_offset + s->sh.slice_cr_qp_offset;
960
961             qp_i = av_clip_c(qp_y + offset, -s->sps->qp_bd_offset, 57);
962             if (qp_i < 30)
963                 qp = qp_i;
964             else if (qp_i > 43)
965                 qp = qp_i - 6;
966             else
967                 qp = qp_c[qp_i - 30];
968
969             qp += s->sps->qp_bd_offset;
970         }
971
972         shift    = s->sps->bit_depth + log2_trafo_size - 5;
973         add      = 1 << (shift - 1);
974         scale    = level_scale[rem6[qp]] << (div6[qp]);
975         scale_m  = 16; // default when no custom scaling lists.
976         dc_scale = 16;
977
978         if (s->sps->scaling_list_enable_flag) {
979             const ScalingList *sl = s->pps->scaling_list_data_present_flag ?
980                                     &s->pps->scaling_list : &s->sps->scaling_list;
981             int matrix_id = lc->cu.pred_mode != MODE_INTRA;
982
983             if (log2_trafo_size != 5)
984                 matrix_id = 3 * matrix_id + c_idx;
985
986             scale_matrix = sl->sl[log2_trafo_size - 2][matrix_id];
987             if (log2_trafo_size >= 4)
988                 dc_scale = sl->sl_dc[log2_trafo_size - 4][matrix_id];
989         }
990     }
991
992     if (s->pps->transform_skip_enabled_flag &&
993         !lc->cu.cu_transquant_bypass_flag   &&
994         log2_trafo_size == 2) {
995         transform_skip_flag = ff_hevc_transform_skip_flag_decode(s, c_idx);
996     }
997
998     last_significant_coeff_x =
999         ff_hevc_last_significant_coeff_x_prefix_decode(s, c_idx, log2_trafo_size);
1000     last_significant_coeff_y =
1001         ff_hevc_last_significant_coeff_y_prefix_decode(s, c_idx, log2_trafo_size);
1002
1003     if (last_significant_coeff_x > 3) {
1004         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_x);
1005         last_significant_coeff_x = (1 << ((last_significant_coeff_x >> 1) - 1)) *
1006                                    (2 + (last_significant_coeff_x & 1)) +
1007                                    suffix;
1008     }
1009
1010     if (last_significant_coeff_y > 3) {
1011         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_y);
1012         last_significant_coeff_y = (1 << ((last_significant_coeff_y >> 1) - 1)) *
1013                                    (2 + (last_significant_coeff_y & 1)) +
1014                                    suffix;
1015     }
1016
1017     if (scan_idx == SCAN_VERT)
1018         FFSWAP(int, last_significant_coeff_x, last_significant_coeff_y);
1019
1020     x_cg_last_sig = last_significant_coeff_x >> 2;
1021     y_cg_last_sig = last_significant_coeff_y >> 2;
1022
1023     switch (scan_idx) {
1024     case SCAN_DIAG: {
1025         int last_x_c = last_significant_coeff_x & 3;
1026         int last_y_c = last_significant_coeff_y & 3;
1027
1028         scan_x_off = ff_hevc_diag_scan4x4_x;
1029         scan_y_off = ff_hevc_diag_scan4x4_y;
1030         num_coeff  = diag_scan4x4_inv[last_y_c][last_x_c];
1031         if (trafo_size == 4) {
1032             scan_x_cg = scan_1x1;
1033             scan_y_cg = scan_1x1;
1034         } else if (trafo_size == 8) {
1035             num_coeff += diag_scan2x2_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1036             scan_x_cg  = diag_scan2x2_x;
1037             scan_y_cg  = diag_scan2x2_y;
1038         } else if (trafo_size == 16) {
1039             num_coeff += diag_scan4x4_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1040             scan_x_cg  = ff_hevc_diag_scan4x4_x;
1041             scan_y_cg  = ff_hevc_diag_scan4x4_y;
1042         } else { // trafo_size == 32
1043             num_coeff += diag_scan8x8_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1044             scan_x_cg  = ff_hevc_diag_scan8x8_x;
1045             scan_y_cg  = ff_hevc_diag_scan8x8_y;
1046         }
1047         break;
1048     }
1049     case SCAN_HORIZ:
1050         scan_x_cg  = horiz_scan2x2_x;
1051         scan_y_cg  = horiz_scan2x2_y;
1052         scan_x_off = horiz_scan4x4_x;
1053         scan_y_off = horiz_scan4x4_y;
1054         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_y][last_significant_coeff_x];
1055         break;
1056     default: //SCAN_VERT
1057         scan_x_cg  = horiz_scan2x2_y;
1058         scan_y_cg  = horiz_scan2x2_x;
1059         scan_x_off = horiz_scan4x4_y;
1060         scan_y_off = horiz_scan4x4_x;
1061         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_x][last_significant_coeff_y];
1062         break;
1063     }
1064     num_coeff++;
1065     num_last_subset = (num_coeff - 1) >> 4;
1066
1067     for (i = num_last_subset; i >= 0; i--) {
1068         int n, m;
1069         int x_cg, y_cg, x_c, y_c;
1070         int implicit_non_zero_coeff = 0;
1071         int64_t trans_coeff_level;
1072         int prev_sig = 0;
1073         int offset   = i << 4;
1074
1075         uint8_t significant_coeff_flag_idx[16];
1076         uint8_t nb_significant_coeff_flag = 0;
1077
1078         x_cg = scan_x_cg[i];
1079         y_cg = scan_y_cg[i];
1080
1081         if (i < num_last_subset && i > 0) {
1082             int ctx_cg = 0;
1083             if (x_cg < (1 << (log2_trafo_size - 2)) - 1)
1084                 ctx_cg += significant_coeff_group_flag[x_cg + 1][y_cg];
1085             if (y_cg < (1 << (log2_trafo_size - 2)) - 1)
1086                 ctx_cg += significant_coeff_group_flag[x_cg][y_cg + 1];
1087
1088             significant_coeff_group_flag[x_cg][y_cg] =
1089                 ff_hevc_significant_coeff_group_flag_decode(s, c_idx, ctx_cg);
1090             implicit_non_zero_coeff = 1;
1091         } else {
1092             significant_coeff_group_flag[x_cg][y_cg] =
1093                 ((x_cg == x_cg_last_sig && y_cg == y_cg_last_sig) ||
1094                  (x_cg == 0 && y_cg == 0));
1095         }
1096
1097         last_scan_pos = num_coeff - offset - 1;
1098
1099         if (i == num_last_subset) {
1100             n_end                         = last_scan_pos - 1;
1101             significant_coeff_flag_idx[0] = last_scan_pos;
1102             nb_significant_coeff_flag     = 1;
1103         } else {
1104             n_end = 15;
1105         }
1106
1107         if (x_cg < ((1 << log2_trafo_size) - 1) >> 2)
1108             prev_sig = significant_coeff_group_flag[x_cg + 1][y_cg];
1109         if (y_cg < ((1 << log2_trafo_size) - 1) >> 2)
1110             prev_sig += significant_coeff_group_flag[x_cg][y_cg + 1] << 1;
1111
1112         for (n = n_end; n >= 0; n--) {
1113             GET_COORD(offset, n);
1114
1115             if (significant_coeff_group_flag[x_cg][y_cg] &&
1116                 (n > 0 || implicit_non_zero_coeff == 0)) {
1117                 if (ff_hevc_significant_coeff_flag_decode(s, c_idx, x_c, y_c,
1118                                                           log2_trafo_size,
1119                                                           scan_idx,
1120                                                           prev_sig) == 1) {
1121                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1122                     nb_significant_coeff_flag++;
1123                     implicit_non_zero_coeff = 0;
1124                 }
1125             } else {
1126                 int last_cg = (x_c == (x_cg << 2) && y_c == (y_cg << 2));
1127                 if (last_cg && implicit_non_zero_coeff && significant_coeff_group_flag[x_cg][y_cg]) {
1128                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1129                     nb_significant_coeff_flag++;
1130                 }
1131             }
1132         }
1133
1134         n_end = nb_significant_coeff_flag;
1135
1136         if (n_end) {
1137             int first_nz_pos_in_cg = 16;
1138             int last_nz_pos_in_cg = -1;
1139             int c_rice_param = 0;
1140             int first_greater1_coeff_idx = -1;
1141             uint8_t coeff_abs_level_greater1_flag[16] = { 0 };
1142             uint16_t coeff_sign_flag;
1143             int sum_abs = 0;
1144             int sign_hidden = 0;
1145
1146             // initialize first elem of coeff_bas_level_greater1_flag
1147             int ctx_set = (i > 0 && c_idx == 0) ? 2 : 0;
1148
1149             if (!(i == num_last_subset) && greater1_ctx == 0)
1150                 ctx_set++;
1151             greater1_ctx      = 1;
1152             last_nz_pos_in_cg = significant_coeff_flag_idx[0];
1153
1154             for (m = 0; m < (n_end > 8 ? 8 : n_end); m++) {
1155                 int n_idx = significant_coeff_flag_idx[m];
1156                 int inc   = (ctx_set << 2) + greater1_ctx;
1157                 coeff_abs_level_greater1_flag[n_idx] =
1158                     ff_hevc_coeff_abs_level_greater1_flag_decode(s, c_idx, inc);
1159                 if (coeff_abs_level_greater1_flag[n_idx]) {
1160                     greater1_ctx = 0;
1161                 } else if (greater1_ctx > 0 && greater1_ctx < 3) {
1162                     greater1_ctx++;
1163                 }
1164
1165                 if (coeff_abs_level_greater1_flag[n_idx] &&
1166                     first_greater1_coeff_idx == -1)
1167                     first_greater1_coeff_idx = n_idx;
1168             }
1169             first_nz_pos_in_cg = significant_coeff_flag_idx[n_end - 1];
1170             sign_hidden        = last_nz_pos_in_cg - first_nz_pos_in_cg >= 4 &&
1171                                  !lc->cu.cu_transquant_bypass_flag;
1172
1173             if (first_greater1_coeff_idx != -1) {
1174                 coeff_abs_level_greater1_flag[first_greater1_coeff_idx] += ff_hevc_coeff_abs_level_greater2_flag_decode(s, c_idx, ctx_set);
1175             }
1176             if (!s->pps->sign_data_hiding_flag || !sign_hidden) {
1177                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag) << (16 - nb_significant_coeff_flag);
1178             } else {
1179                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag - 1) << (16 - (nb_significant_coeff_flag - 1));
1180             }
1181
1182             for (m = 0; m < n_end; m++) {
1183                 n = significant_coeff_flag_idx[m];
1184                 GET_COORD(offset, n);
1185                 trans_coeff_level = 1 + coeff_abs_level_greater1_flag[n];
1186                 if (trans_coeff_level == ((m < 8) ?
1187                                           ((n == first_greater1_coeff_idx) ? 3 : 2) : 1)) {
1188                     int last_coeff_abs_level_remaining = ff_hevc_coeff_abs_level_remaining(s, trans_coeff_level, c_rice_param);
1189
1190                     trans_coeff_level += last_coeff_abs_level_remaining;
1191                     if ((trans_coeff_level) > (3 * (1 << c_rice_param)))
1192                         c_rice_param = FFMIN(c_rice_param + 1, 4);
1193                 }
1194                 if (s->pps->sign_data_hiding_flag && sign_hidden) {
1195                     sum_abs += trans_coeff_level;
1196                     if (n == first_nz_pos_in_cg && ((sum_abs & 1) == 1))
1197                         trans_coeff_level = -trans_coeff_level;
1198                 }
1199                 if (coeff_sign_flag >> 15)
1200                     trans_coeff_level = -trans_coeff_level;
1201                 coeff_sign_flag <<= 1;
1202                 if (!lc->cu.cu_transquant_bypass_flag) {
1203                     if (s->sps->scaling_list_enable_flag) {
1204                         if (y_c || x_c || log2_trafo_size < 4) {
1205                             int pos;
1206                             switch (log2_trafo_size) {
1207                             case 3:  pos = (y_c        << 3) +  x_c;       break;
1208                             case 4:  pos = ((y_c >> 1) << 3) + (x_c >> 1); break;
1209                             case 5:  pos = ((y_c >> 2) << 3) + (x_c >> 2); break;
1210                             default: pos = (y_c        << 2) +  x_c;
1211                             }
1212                             scale_m = scale_matrix[pos];
1213                         } else {
1214                             scale_m = dc_scale;
1215                         }
1216                     }
1217                     trans_coeff_level = (trans_coeff_level * (int64_t)scale * (int64_t)scale_m + add) >> shift;
1218                     if(trans_coeff_level < 0) {
1219                         if((~trans_coeff_level) & 0xFffffffffff8000)
1220                             trans_coeff_level = -32768;
1221                     } else {
1222                         if (trans_coeff_level & 0xffffffffffff8000)
1223                             trans_coeff_level = 32767;
1224                     }
1225                 }
1226                 coeffs[y_c * trafo_size + x_c] = trans_coeff_level;
1227             }
1228         }
1229     }
1230
1231     if (lc->cu.cu_transquant_bypass_flag) {
1232         s->hevcdsp.transquant_bypass[log2_trafo_size - 2](dst, coeffs, stride);
1233     } else {
1234         if (transform_skip_flag)
1235             s->hevcdsp.transform_skip(dst, coeffs, stride);
1236         else if (lc->cu.pred_mode == MODE_INTRA && c_idx == 0 &&
1237                  log2_trafo_size == 2)
1238             s->hevcdsp.transform_4x4_luma_add(dst, coeffs, stride);
1239         else
1240             s->hevcdsp.transform_add[log2_trafo_size - 2](dst, coeffs, stride);
1241     }
1242 }
1243
1244 static int hls_transform_unit(HEVCContext *s, int x0, int y0,
1245                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1246                               int log2_cb_size, int log2_trafo_size,
1247                               int blk_idx, int cbf_luma, int cbf_cb, int cbf_cr)
1248 {
1249     HEVCLocalContext *lc = &s->HEVClc;
1250
1251     if (lc->cu.pred_mode == MODE_INTRA) {
1252         int trafo_size = 1 << log2_trafo_size;
1253         ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1254
1255         s->hpc.intra_pred[log2_trafo_size - 2](s, x0, y0, 0);
1256         if (log2_trafo_size > 2) {
1257             trafo_size = trafo_size << (s->sps->hshift[1] - 1);
1258             ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1259             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 1);
1260             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 2);
1261         } else if (blk_idx == 3) {
1262             trafo_size = trafo_size << s->sps->hshift[1];
1263             ff_hevc_set_neighbour_available(s, xBase, yBase,
1264                                             trafo_size, trafo_size);
1265             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 1);
1266             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 2);
1267         }
1268     }
1269
1270     if (cbf_luma || cbf_cb || cbf_cr) {
1271         int scan_idx   = SCAN_DIAG;
1272         int scan_idx_c = SCAN_DIAG;
1273
1274         if (s->pps->cu_qp_delta_enabled_flag && !lc->tu.is_cu_qp_delta_coded) {
1275             lc->tu.cu_qp_delta = ff_hevc_cu_qp_delta_abs(s);
1276             if (lc->tu.cu_qp_delta != 0)
1277                 if (ff_hevc_cu_qp_delta_sign_flag(s) == 1)
1278                     lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta;
1279             lc->tu.is_cu_qp_delta_coded = 1;
1280
1281             if (lc->tu.cu_qp_delta < -(26 + s->sps->qp_bd_offset / 2) ||
1282                 lc->tu.cu_qp_delta >  (25 + s->sps->qp_bd_offset / 2)) {
1283                 av_log(s->avctx, AV_LOG_ERROR,
1284                        "The cu_qp_delta %d is outside the valid range "
1285                        "[%d, %d].\n",
1286                        lc->tu.cu_qp_delta,
1287                        -(26 + s->sps->qp_bd_offset / 2),
1288                         (25 + s->sps->qp_bd_offset / 2));
1289                 return AVERROR_INVALIDDATA;
1290             }
1291
1292             ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size);
1293         }
1294
1295         if (lc->cu.pred_mode == MODE_INTRA && log2_trafo_size < 4) {
1296             if (lc->tu.cur_intra_pred_mode >= 6 &&
1297                 lc->tu.cur_intra_pred_mode <= 14) {
1298                 scan_idx = SCAN_VERT;
1299             } else if (lc->tu.cur_intra_pred_mode >= 22 &&
1300                        lc->tu.cur_intra_pred_mode <= 30) {
1301                 scan_idx = SCAN_HORIZ;
1302             }
1303
1304             if (lc->pu.intra_pred_mode_c >=  6 &&
1305                 lc->pu.intra_pred_mode_c <= 14) {
1306                 scan_idx_c = SCAN_VERT;
1307             } else if (lc->pu.intra_pred_mode_c >= 22 &&
1308                        lc->pu.intra_pred_mode_c <= 30) {
1309                 scan_idx_c = SCAN_HORIZ;
1310             }
1311         }
1312
1313         if (cbf_luma)
1314             hls_residual_coding(s, x0, y0, log2_trafo_size, scan_idx, 0);
1315         if (log2_trafo_size > 2) {
1316             if (cbf_cb)
1317                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 1);
1318             if (cbf_cr)
1319                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 2);
1320         } else if (blk_idx == 3) {
1321             if (cbf_cb)
1322                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 1);
1323             if (cbf_cr)
1324                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2);
1325         }
1326     }
1327     return 0;
1328 }
1329
1330 static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size)
1331 {
1332     int cb_size          = 1 << log2_cb_size;
1333     int log2_min_pu_size = s->sps->log2_min_pu_size;
1334
1335     int min_pu_width     = s->sps->min_pu_width;
1336     int x_end = FFMIN(x0 + cb_size, s->sps->width);
1337     int y_end = FFMIN(y0 + cb_size, s->sps->height);
1338     int i, j;
1339
1340     for (j = (y0 >> log2_min_pu_size); j < (y_end >> log2_min_pu_size); j++)
1341         for (i = (x0 >> log2_min_pu_size); i < (x_end >> log2_min_pu_size); i++)
1342             s->is_pcm[i + j * min_pu_width] = 2;
1343 }
1344
1345 static int hls_transform_tree(HEVCContext *s, int x0, int y0,
1346                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1347                               int log2_cb_size, int log2_trafo_size,
1348                               int trafo_depth, int blk_idx,
1349                               int cbf_cb, int cbf_cr)
1350 {
1351     HEVCLocalContext *lc = &s->HEVClc;
1352     uint8_t split_transform_flag;
1353     int ret;
1354
1355     if (lc->cu.intra_split_flag) {
1356         if (trafo_depth == 1)
1357             lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[blk_idx];
1358     } else {
1359         lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[0];
1360     }
1361
1362     if (log2_trafo_size <= s->sps->log2_max_trafo_size &&
1363         log2_trafo_size >  s->sps->log2_min_tb_size    &&
1364         trafo_depth     < lc->cu.max_trafo_depth       &&
1365         !(lc->cu.intra_split_flag && trafo_depth == 0)) {
1366         split_transform_flag = ff_hevc_split_transform_flag_decode(s, log2_trafo_size);
1367     } else {
1368         int inter_split = s->sps->max_transform_hierarchy_depth_inter == 0 &&
1369                           lc->cu.pred_mode == MODE_INTER &&
1370                           lc->cu.part_mode != PART_2Nx2N &&
1371                           trafo_depth == 0;
1372
1373         split_transform_flag = log2_trafo_size > s->sps->log2_max_trafo_size ||
1374                                (lc->cu.intra_split_flag && trafo_depth == 0) ||
1375                                inter_split;
1376     }
1377
1378     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cb))
1379         cbf_cb = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1380     else if (log2_trafo_size > 2 || trafo_depth == 0)
1381         cbf_cb = 0;
1382     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cr))
1383         cbf_cr = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1384     else if (log2_trafo_size > 2 || trafo_depth == 0)
1385         cbf_cr = 0;
1386
1387     if (split_transform_flag) {
1388         const int trafo_size_split = 1 << (log2_trafo_size - 1);
1389         const int x1 = x0 + trafo_size_split;
1390         const int y1 = y0 + trafo_size_split;
1391
1392 #define SUBDIVIDE(x, y, idx)                                                    \
1393 do {                                                                            \
1394     ret = hls_transform_tree(s, x, y, x0, y0, cb_xBase, cb_yBase, log2_cb_size, \
1395                              log2_trafo_size - 1, trafo_depth + 1, idx,         \
1396                              cbf_cb, cbf_cr);                                   \
1397     if (ret < 0)                                                                \
1398         return ret;                                                             \
1399 } while (0)
1400
1401         SUBDIVIDE(x0, y0, 0);
1402         SUBDIVIDE(x1, y0, 1);
1403         SUBDIVIDE(x0, y1, 2);
1404         SUBDIVIDE(x1, y1, 3);
1405
1406 #undef SUBDIVIDE
1407     } else {
1408         int min_tu_size      = 1 << s->sps->log2_min_tb_size;
1409         int log2_min_tu_size = s->sps->log2_min_tb_size;
1410         int min_tu_width     = s->sps->min_tb_width;
1411         int cbf_luma         = 1;
1412
1413         if (lc->cu.pred_mode == MODE_INTRA || trafo_depth != 0 ||
1414             cbf_cb || cbf_cr)
1415             cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth);
1416
1417         ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase,
1418                                  log2_cb_size, log2_trafo_size,
1419                                  blk_idx, cbf_luma, cbf_cb, cbf_cr);
1420         if (ret < 0)
1421             return ret;
1422         // TODO: store cbf_luma somewhere else
1423         if (cbf_luma) {
1424             int i, j;
1425             for (i = 0; i < (1 << log2_trafo_size); i += min_tu_size)
1426                 for (j = 0; j < (1 << log2_trafo_size); j += min_tu_size) {
1427                     int x_tu = (x0 + j) >> log2_min_tu_size;
1428                     int y_tu = (y0 + i) >> log2_min_tu_size;
1429                     s->cbf_luma[y_tu * min_tu_width + x_tu] = 1;
1430                 }
1431         }
1432         if (!s->sh.disable_deblocking_filter_flag) {
1433             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_trafo_size);
1434             if (s->pps->transquant_bypass_enable_flag &&
1435                 lc->cu.cu_transquant_bypass_flag)
1436                 set_deblocking_bypass(s, x0, y0, log2_trafo_size);
1437         }
1438     }
1439     return 0;
1440 }
1441
1442 static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size)
1443 {
1444     //TODO: non-4:2:0 support
1445     HEVCLocalContext *lc = &s->HEVClc;
1446     GetBitContext gb;
1447     int cb_size   = 1 << log2_cb_size;
1448     int stride0   = s->frame->linesize[0];
1449     uint8_t *dst0 = &s->frame->data[0][y0 * stride0 + (x0 << s->sps->pixel_shift)];
1450     int   stride1 = s->frame->linesize[1];
1451     uint8_t *dst1 = &s->frame->data[1][(y0 >> s->sps->vshift[1]) * stride1 + ((x0 >> s->sps->hshift[1]) << s->sps->pixel_shift)];
1452     int   stride2 = s->frame->linesize[2];
1453     uint8_t *dst2 = &s->frame->data[2][(y0 >> s->sps->vshift[2]) * stride2 + ((x0 >> s->sps->hshift[2]) << s->sps->pixel_shift)];
1454
1455     int length         = cb_size * cb_size * s->sps->pcm.bit_depth + ((cb_size * cb_size) >> 1) * s->sps->pcm.bit_depth_chroma;
1456     const uint8_t *pcm = skip_bytes(&lc->cc, (length + 7) >> 3);
1457     int ret;
1458
1459     if (!s->sh.disable_deblocking_filter_flag)
1460         ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
1461
1462     ret = init_get_bits(&gb, pcm, length);
1463     if (ret < 0)
1464         return ret;
1465
1466     s->hevcdsp.put_pcm(dst0, stride0, cb_size,     &gb, s->sps->pcm.bit_depth);
1467     s->hevcdsp.put_pcm(dst1, stride1, cb_size / 2, &gb, s->sps->pcm.bit_depth_chroma);
1468     s->hevcdsp.put_pcm(dst2, stride2, cb_size / 2, &gb, s->sps->pcm.bit_depth_chroma);
1469     return 0;
1470 }
1471
1472 static void hls_mvd_coding(HEVCContext *s, int x0, int y0, int log2_cb_size)
1473 {
1474     HEVCLocalContext *lc = &s->HEVClc;
1475     int x = ff_hevc_abs_mvd_greater0_flag_decode(s);
1476     int y = ff_hevc_abs_mvd_greater0_flag_decode(s);
1477
1478     if (x)
1479         x += ff_hevc_abs_mvd_greater1_flag_decode(s);
1480     if (y)
1481         y += ff_hevc_abs_mvd_greater1_flag_decode(s);
1482
1483     switch (x) {
1484     case 2: lc->pu.mvd.x = ff_hevc_mvd_decode(s);           break;
1485     case 1: lc->pu.mvd.x = ff_hevc_mvd_sign_flag_decode(s); break;
1486     case 0: lc->pu.mvd.x = 0;                               break;
1487     }
1488
1489     switch (y) {
1490     case 2: lc->pu.mvd.y = ff_hevc_mvd_decode(s);           break;
1491     case 1: lc->pu.mvd.y = ff_hevc_mvd_sign_flag_decode(s); break;
1492     case 0: lc->pu.mvd.y = 0;                               break;
1493     }
1494 }
1495
1496 /**
1497  * 8.5.3.2.2.1 Luma sample interpolation process
1498  *
1499  * @param s HEVC decoding context
1500  * @param dst target buffer for block data at block position
1501  * @param dststride stride of the dst buffer
1502  * @param ref reference picture buffer at origin (0, 0)
1503  * @param mv motion vector (relative to block position) to get pixel data from
1504  * @param x_off horizontal position of block from origin (0, 0)
1505  * @param y_off vertical position of block from origin (0, 0)
1506  * @param block_w width of block
1507  * @param block_h height of block
1508  */
1509 static void luma_mc(HEVCContext *s, int16_t *dst, ptrdiff_t dststride,
1510                     AVFrame *ref, const Mv *mv, int x_off, int y_off,
1511                     int block_w, int block_h)
1512 {
1513     HEVCLocalContext *lc = &s->HEVClc;
1514     uint8_t *src         = ref->data[0];
1515     ptrdiff_t srcstride  = ref->linesize[0];
1516     int pic_width        = s->sps->width;
1517     int pic_height       = s->sps->height;
1518
1519     int mx         = mv->x & 3;
1520     int my         = mv->y & 3;
1521     int extra_left = ff_hevc_qpel_extra_before[mx];
1522     int extra_top  = ff_hevc_qpel_extra_before[my];
1523
1524     x_off += mv->x >> 2;
1525     y_off += mv->y >> 2;
1526     src   += y_off * srcstride + (x_off << s->sps->pixel_shift);
1527
1528     if (x_off < extra_left || y_off < extra_top ||
1529         x_off >= pic_width - block_w - ff_hevc_qpel_extra_after[mx] ||
1530         y_off >= pic_height - block_h - ff_hevc_qpel_extra_after[my]) {
1531         const int edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->sps->pixel_shift;
1532         int offset = extra_top * srcstride + (extra_left << s->sps->pixel_shift);
1533         int buf_offset = extra_top *
1534                          edge_emu_stride + (extra_left << s->sps->pixel_shift);
1535
1536         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src - offset,
1537                                  edge_emu_stride, srcstride,
1538                                  block_w + ff_hevc_qpel_extra[mx],
1539                                  block_h + ff_hevc_qpel_extra[my],
1540                                  x_off - extra_left, y_off - extra_top,
1541                                  pic_width, pic_height);
1542         src = lc->edge_emu_buffer + buf_offset;
1543         srcstride = edge_emu_stride;
1544     }
1545     s->hevcdsp.put_hevc_qpel[my][mx](dst, dststride, src, srcstride, block_w,
1546                                      block_h, lc->mc_buffer);
1547 }
1548
1549 /**
1550  * 8.5.3.2.2.2 Chroma sample interpolation process
1551  *
1552  * @param s HEVC decoding context
1553  * @param dst1 target buffer for block data at block position (U plane)
1554  * @param dst2 target buffer for block data at block position (V plane)
1555  * @param dststride stride of the dst1 and dst2 buffers
1556  * @param ref reference picture buffer at origin (0, 0)
1557  * @param mv motion vector (relative to block position) to get pixel data from
1558  * @param x_off horizontal position of block from origin (0, 0)
1559  * @param y_off vertical position of block from origin (0, 0)
1560  * @param block_w width of block
1561  * @param block_h height of block
1562  */
1563 static void chroma_mc(HEVCContext *s, int16_t *dst1, int16_t *dst2,
1564                       ptrdiff_t dststride, AVFrame *ref, const Mv *mv,
1565                       int x_off, int y_off, int block_w, int block_h)
1566 {
1567     HEVCLocalContext *lc = &s->HEVClc;
1568     uint8_t *src1        = ref->data[1];
1569     uint8_t *src2        = ref->data[2];
1570     ptrdiff_t src1stride = ref->linesize[1];
1571     ptrdiff_t src2stride = ref->linesize[2];
1572     int pic_width        = s->sps->width >> 1;
1573     int pic_height       = s->sps->height >> 1;
1574
1575     int mx = mv->x & 7;
1576     int my = mv->y & 7;
1577
1578     x_off += mv->x >> 3;
1579     y_off += mv->y >> 3;
1580     src1  += y_off * src1stride + (x_off << s->sps->pixel_shift);
1581     src2  += y_off * src2stride + (x_off << s->sps->pixel_shift);
1582
1583     if (x_off < EPEL_EXTRA_BEFORE || y_off < EPEL_EXTRA_AFTER ||
1584         x_off >= pic_width - block_w - EPEL_EXTRA_AFTER ||
1585         y_off >= pic_height - block_h - EPEL_EXTRA_AFTER) {
1586         const int edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->sps->pixel_shift;
1587         int offset1 = EPEL_EXTRA_BEFORE * (src1stride + (1 << s->sps->pixel_shift));
1588         int buf_offset1 = EPEL_EXTRA_BEFORE *
1589                           (edge_emu_stride + (1 << s->sps->pixel_shift));
1590         int offset2 = EPEL_EXTRA_BEFORE * (src2stride + (1 << s->sps->pixel_shift));
1591         int buf_offset2 = EPEL_EXTRA_BEFORE *
1592                           (edge_emu_stride + (1 << s->sps->pixel_shift));
1593
1594         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src1 - offset1,
1595                                  edge_emu_stride, src1stride,
1596                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1597                                  x_off - EPEL_EXTRA_BEFORE,
1598                                  y_off - EPEL_EXTRA_BEFORE,
1599                                  pic_width, pic_height);
1600
1601         src1 = lc->edge_emu_buffer + buf_offset1;
1602         src1stride = edge_emu_stride;
1603         s->hevcdsp.put_hevc_epel[!!my][!!mx](dst1, dststride, src1, src1stride,
1604                                              block_w, block_h, mx, my, lc->mc_buffer);
1605
1606         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src2 - offset2,
1607                                  edge_emu_stride, src2stride,
1608                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1609                                  x_off - EPEL_EXTRA_BEFORE,
1610                                  y_off - EPEL_EXTRA_BEFORE,
1611                                  pic_width, pic_height);
1612         src2 = lc->edge_emu_buffer + buf_offset2;
1613         src2stride = edge_emu_stride;
1614
1615         s->hevcdsp.put_hevc_epel[!!my][!!mx](dst2, dststride, src2, src2stride,
1616                                              block_w, block_h, mx, my,
1617                                              lc->mc_buffer);
1618     } else {
1619         s->hevcdsp.put_hevc_epel[!!my][!!mx](dst1, dststride, src1, src1stride,
1620                                              block_w, block_h, mx, my,
1621                                              lc->mc_buffer);
1622         s->hevcdsp.put_hevc_epel[!!my][!!mx](dst2, dststride, src2, src2stride,
1623                                              block_w, block_h, mx, my,
1624                                              lc->mc_buffer);
1625     }
1626 }
1627
1628 static void hevc_await_progress(HEVCContext *s, HEVCFrame *ref,
1629                                 const Mv *mv, int y0, int height)
1630 {
1631     int y = (mv->y >> 2) + y0 + height + 9;
1632     ff_thread_await_progress(&ref->tf, y, 0);
1633 }
1634
1635 static void hevc_luma_mv_mpv_mode(HEVCContext *s, int x0, int y0, int nPbW,
1636                                   int nPbH, int log2_cb_size, int part_idx,
1637                                   int merge_idx, MvField *mv)
1638 {
1639     HEVCLocalContext *lc             = &s->HEVClc;
1640     enum InterPredIdc inter_pred_idc = PRED_L0;
1641     int mvp_flag;
1642
1643     ff_hevc_set_neighbour_available(s, x0, y0, nPbW, nPbH);
1644     if (s->sh.slice_type == B_SLICE)
1645         inter_pred_idc = ff_hevc_inter_pred_idc_decode(s, nPbW, nPbH);
1646
1647     if (inter_pred_idc != PRED_L1) {
1648         if (s->sh.nb_refs[L0])
1649             mv->ref_idx[0]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L0]);
1650
1651         mv->pred_flag[0] = 1;
1652         hls_mvd_coding(s, x0, y0, 0);
1653         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1654         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1655                                  part_idx, merge_idx, mv, mvp_flag, 0);
1656         mv->mv[0].x += lc->pu.mvd.x;
1657         mv->mv[0].y += lc->pu.mvd.y;
1658     }
1659
1660     if (inter_pred_idc != PRED_L0) {
1661         if (s->sh.nb_refs[L1])
1662             mv->ref_idx[1]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L1]);
1663
1664         if (s->sh.mvd_l1_zero_flag == 1 && inter_pred_idc == PRED_BI) {
1665             AV_ZERO32(&lc->pu.mvd);
1666         } else {
1667             hls_mvd_coding(s, x0, y0, 1);
1668         }
1669
1670         mv->pred_flag[1] = 1;
1671         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1672         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1673                                  part_idx, merge_idx, mv, mvp_flag, 1);
1674         mv->mv[1].x += lc->pu.mvd.x;
1675         mv->mv[1].y += lc->pu.mvd.y;
1676     }
1677 }
1678
1679 static void hls_prediction_unit(HEVCContext *s, int x0, int y0,
1680                                 int nPbW, int nPbH,
1681                                 int log2_cb_size, int partIdx)
1682 {
1683 #define POS(c_idx, x, y)                                                              \
1684     &s->frame->data[c_idx][((y) >> s->sps->vshift[c_idx]) * s->frame->linesize[c_idx] + \
1685                            (((x) >> s->sps->hshift[c_idx]) << s->sps->pixel_shift)]
1686     HEVCLocalContext *lc = &s->HEVClc;
1687     int merge_idx = 0;
1688     struct MvField current_mv = {{{ 0 }}};
1689
1690     int min_pu_width = s->sps->min_pu_width;
1691
1692     MvField *tab_mvf = s->ref->tab_mvf;
1693     RefPicList  *refPicList = s->ref->refPicList;
1694     HEVCFrame *ref0, *ref1;
1695
1696     int tmpstride = MAX_PB_SIZE;
1697
1698     uint8_t *dst0 = POS(0, x0, y0);
1699     uint8_t *dst1 = POS(1, x0, y0);
1700     uint8_t *dst2 = POS(2, x0, y0);
1701     int log2_min_cb_size = s->sps->log2_min_cb_size;
1702     int min_cb_width     = s->sps->min_cb_width;
1703     int x_cb             = x0 >> log2_min_cb_size;
1704     int y_cb             = y0 >> log2_min_cb_size;
1705     int x_pu, y_pu;
1706     int i, j;
1707
1708     int skip_flag = SAMPLE_CTB(s->skip_flag, x_cb, y_cb);
1709
1710     if (!skip_flag)
1711         lc->pu.merge_flag = ff_hevc_merge_flag_decode(s);
1712
1713     if (skip_flag || lc->pu.merge_flag) {
1714         if (s->sh.max_num_merge_cand > 1)
1715             merge_idx = ff_hevc_merge_idx_decode(s);
1716         else
1717             merge_idx = 0;
1718
1719         ff_hevc_luma_mv_merge_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1720                                    partIdx, merge_idx, &current_mv);
1721     } else {
1722         hevc_luma_mv_mpv_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1723                               partIdx, merge_idx, &current_mv);
1724     }
1725
1726     x_pu = x0 >> s->sps->log2_min_pu_size;
1727     y_pu = y0 >> s->sps->log2_min_pu_size;
1728
1729     for (j = 0; j < nPbH >> s->sps->log2_min_pu_size; j++)
1730         for (i = 0; i < nPbW >> s->sps->log2_min_pu_size; i++)
1731             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i] = current_mv;
1732
1733     if (current_mv.pred_flag[0]) {
1734         ref0 = refPicList[0].ref[current_mv.ref_idx[0]];
1735         if (!ref0)
1736             return;
1737         hevc_await_progress(s, ref0, &current_mv.mv[0], y0, nPbH);
1738     }
1739     if (current_mv.pred_flag[1]) {
1740         ref1 = refPicList[1].ref[current_mv.ref_idx[1]];
1741         if (!ref1)
1742             return;
1743         hevc_await_progress(s, ref1, &current_mv.mv[1], y0, nPbH);
1744     }
1745
1746     if (current_mv.pred_flag[0] && !current_mv.pred_flag[1]) {
1747         DECLARE_ALIGNED(16, int16_t,  tmp[MAX_PB_SIZE * MAX_PB_SIZE]);
1748         DECLARE_ALIGNED(16, int16_t, tmp2[MAX_PB_SIZE * MAX_PB_SIZE]);
1749
1750         luma_mc(s, tmp, tmpstride, ref0->frame,
1751                 &current_mv.mv[0], x0, y0, nPbW, nPbH);
1752
1753         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1754             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1755             s->hevcdsp.weighted_pred(s->sh.luma_log2_weight_denom,
1756                                      s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1757                                      s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1758                                      dst0, s->frame->linesize[0], tmp,
1759                                      tmpstride, nPbW, nPbH);
1760         } else {
1761             s->hevcdsp.put_unweighted_pred(dst0, s->frame->linesize[0], tmp, tmpstride, nPbW, nPbH);
1762         }
1763         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1764                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2);
1765
1766         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1767             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1768             s->hevcdsp.weighted_pred(s->sh.chroma_log2_weight_denom,
1769                                      s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1770                                      s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1771                                      dst1, s->frame->linesize[1], tmp, tmpstride,
1772                                      nPbW / 2, nPbH / 2);
1773             s->hevcdsp.weighted_pred(s->sh.chroma_log2_weight_denom,
1774                                      s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1775                                      s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1776                                      dst2, s->frame->linesize[2], tmp2, tmpstride,
1777                                      nPbW / 2, nPbH / 2);
1778         } else {
1779             s->hevcdsp.put_unweighted_pred(dst1, s->frame->linesize[1], tmp, tmpstride, nPbW/2, nPbH/2);
1780             s->hevcdsp.put_unweighted_pred(dst2, s->frame->linesize[2], tmp2, tmpstride, nPbW/2, nPbH/2);
1781         }
1782     } else if (!current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1783         DECLARE_ALIGNED(16, int16_t, tmp [MAX_PB_SIZE * MAX_PB_SIZE]);
1784         DECLARE_ALIGNED(16, int16_t, tmp2[MAX_PB_SIZE * MAX_PB_SIZE]);
1785
1786         luma_mc(s, tmp, tmpstride, ref1->frame,
1787                 &current_mv.mv[1], x0, y0, nPbW, nPbH);
1788
1789         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1790             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1791             s->hevcdsp.weighted_pred(s->sh.luma_log2_weight_denom,
1792                                       s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1793                                       s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1794                                       dst0, s->frame->linesize[0], tmp, tmpstride,
1795                                       nPbW, nPbH);
1796         } else {
1797             s->hevcdsp.put_unweighted_pred(dst0, s->frame->linesize[0], tmp, tmpstride, nPbW, nPbH);
1798         }
1799
1800         chroma_mc(s, tmp, tmp2, tmpstride, ref1->frame,
1801                   &current_mv.mv[1], x0/2, y0/2, nPbW/2, nPbH/2);
1802
1803         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1804             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1805             s->hevcdsp.weighted_pred(s->sh.chroma_log2_weight_denom,
1806                                      s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1807                                      s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1808                                      dst1, s->frame->linesize[1], tmp, tmpstride, nPbW/2, nPbH/2);
1809             s->hevcdsp.weighted_pred(s->sh.chroma_log2_weight_denom,
1810                                      s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1811                                      s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1812                                      dst2, s->frame->linesize[2], tmp2, tmpstride, nPbW/2, nPbH/2);
1813         } else {
1814             s->hevcdsp.put_unweighted_pred(dst1, s->frame->linesize[1], tmp, tmpstride, nPbW/2, nPbH/2);
1815             s->hevcdsp.put_unweighted_pred(dst2, s->frame->linesize[2], tmp2, tmpstride, nPbW/2, nPbH/2);
1816         }
1817     } else if (current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1818         DECLARE_ALIGNED(16, int16_t, tmp [MAX_PB_SIZE * MAX_PB_SIZE]);
1819         DECLARE_ALIGNED(16, int16_t, tmp2[MAX_PB_SIZE * MAX_PB_SIZE]);
1820         DECLARE_ALIGNED(16, int16_t, tmp3[MAX_PB_SIZE * MAX_PB_SIZE]);
1821         DECLARE_ALIGNED(16, int16_t, tmp4[MAX_PB_SIZE * MAX_PB_SIZE]);
1822
1823         luma_mc(s, tmp, tmpstride, ref0->frame,
1824                 &current_mv.mv[0], x0, y0, nPbW, nPbH);
1825         luma_mc(s, tmp2, tmpstride, ref1->frame,
1826                 &current_mv.mv[1], x0, y0, nPbW, nPbH);
1827
1828         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1829             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1830             s->hevcdsp.weighted_pred_avg(s->sh.luma_log2_weight_denom,
1831                                          s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1832                                          s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1833                                          s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1834                                          s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1835                                          dst0, s->frame->linesize[0],
1836                                          tmp, tmp2, tmpstride, nPbW, nPbH);
1837         } else {
1838             s->hevcdsp.put_weighted_pred_avg(dst0, s->frame->linesize[0],
1839                                              tmp, tmp2, tmpstride, nPbW, nPbH);
1840         }
1841
1842         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1843                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2);
1844         chroma_mc(s, tmp3, tmp4, tmpstride, ref1->frame,
1845                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2);
1846
1847         if ((s->sh.slice_type == P_SLICE && s->pps->weighted_pred_flag) ||
1848             (s->sh.slice_type == B_SLICE && s->pps->weighted_bipred_flag)) {
1849             s->hevcdsp.weighted_pred_avg(s->sh.chroma_log2_weight_denom,
1850                                          s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1851                                          s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1852                                          s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1853                                          s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1854                                          dst1, s->frame->linesize[1], tmp, tmp3,
1855                                          tmpstride, nPbW / 2, nPbH / 2);
1856             s->hevcdsp.weighted_pred_avg(s->sh.chroma_log2_weight_denom,
1857                                          s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1858                                          s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1859                                          s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1860                                          s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1861                                          dst2, s->frame->linesize[2], tmp2, tmp4,
1862                                          tmpstride, nPbW / 2, nPbH / 2);
1863         } else {
1864             s->hevcdsp.put_weighted_pred_avg(dst1, s->frame->linesize[1], tmp, tmp3, tmpstride, nPbW/2, nPbH/2);
1865             s->hevcdsp.put_weighted_pred_avg(dst2, s->frame->linesize[2], tmp2, tmp4, tmpstride, nPbW/2, nPbH/2);
1866         }
1867     }
1868 }
1869
1870 /**
1871  * 8.4.1
1872  */
1873 static int luma_intra_pred_mode(HEVCContext *s, int x0, int y0, int pu_size,
1874                                 int prev_intra_luma_pred_flag)
1875 {
1876     HEVCLocalContext *lc = &s->HEVClc;
1877     int x_pu             = x0 >> s->sps->log2_min_pu_size;
1878     int y_pu             = y0 >> s->sps->log2_min_pu_size;
1879     int min_pu_width     = s->sps->min_pu_width;
1880     int size_in_pus      = pu_size >> s->sps->log2_min_pu_size;
1881     int x0b              = x0 & ((1 << s->sps->log2_ctb_size) - 1);
1882     int y0b              = y0 & ((1 << s->sps->log2_ctb_size) - 1);
1883
1884     int cand_up   = (lc->ctb_up_flag || y0b) ?
1885                     s->tab_ipm[(y_pu - 1) * min_pu_width + x_pu] : INTRA_DC;
1886     int cand_left = (lc->ctb_left_flag || x0b) ?
1887                     s->tab_ipm[y_pu * min_pu_width + x_pu - 1]   : INTRA_DC;
1888
1889     int y_ctb = (y0 >> (s->sps->log2_ctb_size)) << (s->sps->log2_ctb_size);
1890
1891     MvField *tab_mvf = s->ref->tab_mvf;
1892     int intra_pred_mode;
1893     int candidate[3];
1894     int i, j;
1895
1896     // intra_pred_mode prediction does not cross vertical CTB boundaries
1897     if ((y0 - 1) < y_ctb)
1898         cand_up = INTRA_DC;
1899
1900     if (cand_left == cand_up) {
1901         if (cand_left < 2) {
1902             candidate[0] = INTRA_PLANAR;
1903             candidate[1] = INTRA_DC;
1904             candidate[2] = INTRA_ANGULAR_26;
1905         } else {
1906             candidate[0] = cand_left;
1907             candidate[1] = 2 + ((cand_left - 2 - 1 + 32) & 31);
1908             candidate[2] = 2 + ((cand_left - 2 + 1) & 31);
1909         }
1910     } else {
1911         candidate[0] = cand_left;
1912         candidate[1] = cand_up;
1913         if (candidate[0] != INTRA_PLANAR && candidate[1] != INTRA_PLANAR) {
1914             candidate[2] = INTRA_PLANAR;
1915         } else if (candidate[0] != INTRA_DC && candidate[1] != INTRA_DC) {
1916             candidate[2] = INTRA_DC;
1917         } else {
1918             candidate[2] = INTRA_ANGULAR_26;
1919         }
1920     }
1921
1922     if (prev_intra_luma_pred_flag) {
1923         intra_pred_mode = candidate[lc->pu.mpm_idx];
1924     } else {
1925         if (candidate[0] > candidate[1])
1926             FFSWAP(uint8_t, candidate[0], candidate[1]);
1927         if (candidate[0] > candidate[2])
1928             FFSWAP(uint8_t, candidate[0], candidate[2]);
1929         if (candidate[1] > candidate[2])
1930             FFSWAP(uint8_t, candidate[1], candidate[2]);
1931
1932         intra_pred_mode = lc->pu.rem_intra_luma_pred_mode;
1933         for (i = 0; i < 3; i++)
1934             if (intra_pred_mode >= candidate[i])
1935                 intra_pred_mode++;
1936     }
1937
1938     /* write the intra prediction units into the mv array */
1939     if (!size_in_pus)
1940         size_in_pus = 1;
1941     for (i = 0; i < size_in_pus; i++) {
1942         memset(&s->tab_ipm[(y_pu + i) * min_pu_width + x_pu],
1943                intra_pred_mode, size_in_pus);
1944
1945         for (j = 0; j < size_in_pus; j++) {
1946             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].is_intra     = 1;
1947             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[0] = 0;
1948             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[1] = 0;
1949             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[0]   = 0;
1950             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[1]   = 0;
1951             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].x      = 0;
1952             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].y      = 0;
1953             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].x      = 0;
1954             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].y      = 0;
1955         }
1956     }
1957
1958     return intra_pred_mode;
1959 }
1960
1961 static av_always_inline void set_ct_depth(HEVCContext *s, int x0, int y0,
1962                                           int log2_cb_size, int ct_depth)
1963 {
1964     int length = (1 << log2_cb_size) >> s->sps->log2_min_cb_size;
1965     int x_cb   = x0 >> s->sps->log2_min_cb_size;
1966     int y_cb   = y0 >> s->sps->log2_min_cb_size;
1967     int y;
1968
1969     for (y = 0; y < length; y++)
1970         memset(&s->tab_ct_depth[(y_cb + y) * s->sps->min_cb_width + x_cb],
1971                ct_depth, length);
1972 }
1973
1974 static void intra_prediction_unit(HEVCContext *s, int x0, int y0,
1975                                   int log2_cb_size)
1976 {
1977     HEVCLocalContext *lc = &s->HEVClc;
1978     static const uint8_t intra_chroma_table[4] = { 0, 26, 10, 1 };
1979     uint8_t prev_intra_luma_pred_flag[4];
1980     int split   = lc->cu.part_mode == PART_NxN;
1981     int pb_size = (1 << log2_cb_size) >> split;
1982     int side    = split + 1;
1983     int chroma_mode;
1984     int i, j;
1985
1986     for (i = 0; i < side; i++)
1987         for (j = 0; j < side; j++)
1988             prev_intra_luma_pred_flag[2 * i + j] = ff_hevc_prev_intra_luma_pred_flag_decode(s);
1989
1990     for (i = 0; i < side; i++) {
1991         for (j = 0; j < side; j++) {
1992             if (prev_intra_luma_pred_flag[2 * i + j])
1993                 lc->pu.mpm_idx = ff_hevc_mpm_idx_decode(s);
1994             else
1995                 lc->pu.rem_intra_luma_pred_mode = ff_hevc_rem_intra_luma_pred_mode_decode(s);
1996
1997             lc->pu.intra_pred_mode[2 * i + j] =
1998                 luma_intra_pred_mode(s, x0 + pb_size * j, y0 + pb_size * i, pb_size,
1999                                      prev_intra_luma_pred_flag[2 * i + j]);
2000         }
2001     }
2002
2003     chroma_mode = ff_hevc_intra_chroma_pred_mode_decode(s);
2004     if (chroma_mode != 4) {
2005         if (lc->pu.intra_pred_mode[0] == intra_chroma_table[chroma_mode])
2006             lc->pu.intra_pred_mode_c = 34;
2007         else
2008             lc->pu.intra_pred_mode_c = intra_chroma_table[chroma_mode];
2009     } else {
2010         lc->pu.intra_pred_mode_c = lc->pu.intra_pred_mode[0];
2011     }
2012 }
2013
2014 static void intra_prediction_unit_default_value(HEVCContext *s,
2015                                                 int x0, int y0,
2016                                                 int log2_cb_size)
2017 {
2018     HEVCLocalContext *lc = &s->HEVClc;
2019     int pb_size          = 1 << log2_cb_size;
2020     int size_in_pus      = pb_size >> s->sps->log2_min_pu_size;
2021     int min_pu_width     = s->sps->min_pu_width;
2022     MvField *tab_mvf     = s->ref->tab_mvf;
2023     int x_pu             = x0 >> s->sps->log2_min_pu_size;
2024     int y_pu             = y0 >> s->sps->log2_min_pu_size;
2025     int j, k;
2026
2027     if (size_in_pus == 0)
2028         size_in_pus = 1;
2029     for (j = 0; j < size_in_pus; j++) {
2030         memset(&s->tab_ipm[(y_pu + j) * min_pu_width + x_pu], INTRA_DC, size_in_pus);
2031         for (k = 0; k < size_in_pus; k++)
2032             tab_mvf[(y_pu + j) * min_pu_width + x_pu + k].is_intra = lc->cu.pred_mode == MODE_INTRA;
2033     }
2034 }
2035
2036 static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
2037 {
2038     int cb_size          = 1 << log2_cb_size;
2039     HEVCLocalContext *lc = &s->HEVClc;
2040     int log2_min_cb_size = s->sps->log2_min_cb_size;
2041     int length           = cb_size >> log2_min_cb_size;
2042     int min_cb_width     = s->sps->min_cb_width;
2043     int x_cb             = x0 >> log2_min_cb_size;
2044     int y_cb             = y0 >> log2_min_cb_size;
2045     int x, y, ret;
2046
2047     lc->cu.x                = x0;
2048     lc->cu.y                = y0;
2049     lc->cu.pred_mode        = MODE_INTRA;
2050     lc->cu.part_mode        = PART_2Nx2N;
2051     lc->cu.intra_split_flag = 0;
2052
2053     SAMPLE_CTB(s->skip_flag, x_cb, y_cb) = 0;
2054     for (x = 0; x < 4; x++)
2055         lc->pu.intra_pred_mode[x] = 1;
2056     if (s->pps->transquant_bypass_enable_flag) {
2057         lc->cu.cu_transquant_bypass_flag = ff_hevc_cu_transquant_bypass_flag_decode(s);
2058         if (lc->cu.cu_transquant_bypass_flag)
2059             set_deblocking_bypass(s, x0, y0, log2_cb_size);
2060     } else
2061         lc->cu.cu_transquant_bypass_flag = 0;
2062
2063     if (s->sh.slice_type != I_SLICE) {
2064         uint8_t skip_flag = ff_hevc_skip_flag_decode(s, x0, y0, x_cb, y_cb);
2065
2066         x = y_cb * min_cb_width + x_cb;
2067         for (y = 0; y < length; y++) {
2068             memset(&s->skip_flag[x], skip_flag, length);
2069             x += min_cb_width;
2070         }
2071         lc->cu.pred_mode = skip_flag ? MODE_SKIP : MODE_INTER;
2072     }
2073
2074     if (SAMPLE_CTB(s->skip_flag, x_cb, y_cb)) {
2075         hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2076         intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2077
2078         if (!s->sh.disable_deblocking_filter_flag)
2079             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2080     } else {
2081         int pcm_flag = 0;
2082
2083         if (s->sh.slice_type != I_SLICE)
2084             lc->cu.pred_mode = ff_hevc_pred_mode_decode(s);
2085         if (lc->cu.pred_mode != MODE_INTRA ||
2086             log2_cb_size == s->sps->log2_min_cb_size) {
2087             lc->cu.part_mode        = ff_hevc_part_mode_decode(s, log2_cb_size);
2088             lc->cu.intra_split_flag = lc->cu.part_mode == PART_NxN &&
2089                                       lc->cu.pred_mode == MODE_INTRA;
2090         }
2091
2092         if (lc->cu.pred_mode == MODE_INTRA) {
2093             if (lc->cu.part_mode == PART_2Nx2N && s->sps->pcm_enabled_flag &&
2094                 log2_cb_size >= s->sps->pcm.log2_min_pcm_cb_size &&
2095                 log2_cb_size <= s->sps->pcm.log2_max_pcm_cb_size) {
2096                 pcm_flag = ff_hevc_pcm_flag_decode(s);
2097             }
2098             if (pcm_flag) {
2099                 intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2100                 ret = hls_pcm_sample(s, x0, y0, log2_cb_size);
2101                 if (s->sps->pcm.loop_filter_disable_flag)
2102                     set_deblocking_bypass(s, x0, y0, log2_cb_size);
2103
2104                 if (ret < 0)
2105                     return ret;
2106             } else {
2107                 intra_prediction_unit(s, x0, y0, log2_cb_size);
2108             }
2109         } else {
2110             intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2111             switch (lc->cu.part_mode) {
2112             case PART_2Nx2N:
2113                 hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2114                 break;
2115             case PART_2NxN:
2116                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size / 2, log2_cb_size, 0);
2117                 hls_prediction_unit(s, x0, y0 + cb_size / 2, cb_size, cb_size / 2, log2_cb_size, 1);
2118                 break;
2119             case PART_Nx2N:
2120                 hls_prediction_unit(s, x0,               y0, cb_size / 2, cb_size, log2_cb_size, 0);
2121                 hls_prediction_unit(s, x0 + cb_size / 2, y0, cb_size / 2, cb_size, log2_cb_size, 1);
2122                 break;
2123             case PART_2NxnU:
2124                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size     / 4, log2_cb_size, 0);
2125                 hls_prediction_unit(s, x0, y0 + cb_size / 4, cb_size, cb_size * 3 / 4, log2_cb_size, 1);
2126                 break;
2127             case PART_2NxnD:
2128                 hls_prediction_unit(s, x0, y0,                   cb_size, cb_size * 3 / 4, log2_cb_size, 0);
2129                 hls_prediction_unit(s, x0, y0 + cb_size * 3 / 4, cb_size, cb_size     / 4, log2_cb_size, 1);
2130                 break;
2131             case PART_nLx2N:
2132                 hls_prediction_unit(s, x0,               y0, cb_size     / 4, cb_size, log2_cb_size, 0);
2133                 hls_prediction_unit(s, x0 + cb_size / 4, y0, cb_size * 3 / 4, cb_size, log2_cb_size, 1);
2134                 break;
2135             case PART_nRx2N:
2136                 hls_prediction_unit(s, x0,                   y0, cb_size * 3 / 4, cb_size, log2_cb_size, 0);
2137                 hls_prediction_unit(s, x0 + cb_size * 3 / 4, y0, cb_size     / 4, cb_size, log2_cb_size, 1);
2138                 break;
2139             case PART_NxN:
2140                 hls_prediction_unit(s, x0,               y0,               cb_size / 2, cb_size / 2, log2_cb_size, 0);
2141                 hls_prediction_unit(s, x0 + cb_size / 2, y0,               cb_size / 2, cb_size / 2, log2_cb_size, 1);
2142                 hls_prediction_unit(s, x0,               y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 2);
2143                 hls_prediction_unit(s, x0 + cb_size / 2, y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 3);
2144                 break;
2145             }
2146         }
2147
2148         if (!pcm_flag) {
2149             int rqt_root_cbf = 1;
2150
2151             if (lc->cu.pred_mode != MODE_INTRA &&
2152                 !(lc->cu.part_mode == PART_2Nx2N && lc->pu.merge_flag)) {
2153                 rqt_root_cbf = ff_hevc_no_residual_syntax_flag_decode(s);
2154             }
2155             if (rqt_root_cbf) {
2156                 lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ?
2157                                          s->sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag :
2158                                          s->sps->max_transform_hierarchy_depth_inter;
2159                 ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0,
2160                                          log2_cb_size,
2161                                          log2_cb_size, 0, 0, 0, 0);
2162                 if (ret < 0)
2163                     return ret;
2164             } else {
2165                 if (!s->sh.disable_deblocking_filter_flag)
2166                     ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2167             }
2168         }
2169     }
2170
2171     if (s->pps->cu_qp_delta_enabled_flag && lc->tu.is_cu_qp_delta_coded == 0)
2172         ff_hevc_set_qPy(s, x0, y0, x0, y0, log2_cb_size);
2173
2174     x = y_cb * min_cb_width + x_cb;
2175     for (y = 0; y < length; y++) {
2176         memset(&s->qp_y_tab[x], lc->qp_y, length);
2177         x += min_cb_width;
2178     }
2179
2180     set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth);
2181
2182     return 0;
2183 }
2184
2185 static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
2186                                int log2_cb_size, int cb_depth)
2187 {
2188     HEVCLocalContext *lc = &s->HEVClc;
2189     const int cb_size    = 1 << log2_cb_size;
2190     int split_cu;
2191
2192     lc->ct.depth = cb_depth;
2193     if (x0 + cb_size <= s->sps->width  &&
2194         y0 + cb_size <= s->sps->height &&
2195         log2_cb_size > s->sps->log2_min_cb_size) {
2196         split_cu = ff_hevc_split_coding_unit_flag_decode(s, cb_depth, x0, y0);
2197     } else {
2198         split_cu = (log2_cb_size > s->sps->log2_min_cb_size);
2199     }
2200     if (s->pps->cu_qp_delta_enabled_flag &&
2201         log2_cb_size >= s->sps->log2_ctb_size - s->pps->diff_cu_qp_delta_depth) {
2202         lc->tu.is_cu_qp_delta_coded = 0;
2203         lc->tu.cu_qp_delta          = 0;
2204     }
2205
2206     if (split_cu) {
2207         const int cb_size_split = cb_size >> 1;
2208         const int x1 = x0 + cb_size_split;
2209         const int y1 = y0 + cb_size_split;
2210
2211         log2_cb_size--;
2212         cb_depth++;
2213
2214 #define SUBDIVIDE(x, y)                                                \
2215 do {                                                                   \
2216     if (x < s->sps->width && y < s->sps->height) {                     \
2217         int ret = hls_coding_quadtree(s, x, y, log2_cb_size, cb_depth);\
2218         if (ret < 0)                                                   \
2219             return ret;                                                \
2220     }                                                                  \
2221 } while (0)
2222
2223         SUBDIVIDE(x0, y0);
2224         SUBDIVIDE(x1, y0);
2225         SUBDIVIDE(x0, y1);
2226         SUBDIVIDE(x1, y1);
2227     } else {
2228         int ret = hls_coding_unit(s, x0, y0, log2_cb_size);
2229         if (ret < 0)
2230             return ret;
2231     }
2232
2233     return 0;
2234 }
2235
2236 static void hls_decode_neighbour(HEVCContext *s, int x_ctb, int y_ctb,
2237                                  int ctb_addr_ts)
2238 {
2239     HEVCLocalContext *lc  = &s->HEVClc;
2240     int ctb_size          = 1 << s->sps->log2_ctb_size;
2241     int ctb_addr_rs       = s->pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2242     int ctb_addr_in_slice = ctb_addr_rs - s->sh.slice_addr;
2243
2244     s->tab_slice_address[ctb_addr_rs] = s->sh.slice_addr;
2245
2246     if (s->pps->entropy_coding_sync_enabled_flag) {
2247         if (x_ctb == 0 && (y_ctb & (ctb_size - 1)) == 0)
2248             lc->first_qp_group = 1;
2249         lc->end_of_tiles_x = s->sps->width;
2250     } else if (s->pps->tiles_enabled_flag) {
2251         if (ctb_addr_ts && s->pps->tile_id[ctb_addr_ts] != s->pps->tile_id[ctb_addr_ts - 1]) {
2252             int idxX = s->pps->col_idxX[x_ctb >> s->sps->log2_ctb_size];
2253             lc->start_of_tiles_x = x_ctb;
2254             lc->end_of_tiles_x   = x_ctb + (s->pps->column_width[idxX] << s->sps->log2_ctb_size);
2255             lc->first_qp_group   = 1;
2256         }
2257     } else {
2258         lc->end_of_tiles_x = s->sps->width;
2259     }
2260
2261     lc->end_of_tiles_y = FFMIN(y_ctb + ctb_size, s->sps->height);
2262
2263     lc->boundary_flags = 0;
2264     if (s->pps->tiles_enabled_flag) {
2265         if (x_ctb > 0 && s->pps->tile_id[ctb_addr_ts] != s->pps->tile_id[s->pps->ctb_addr_rs_to_ts[ctb_addr_rs - 1]])
2266             lc->boundary_flags |= BOUNDARY_LEFT_TILE;
2267         if (x_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - 1])
2268             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2269         if (y_ctb > 0 && s->pps->tile_id[ctb_addr_ts] != s->pps->tile_id[s->pps->ctb_addr_rs_to_ts[ctb_addr_rs - s->sps->ctb_width]])
2270             lc->boundary_flags |= BOUNDARY_UPPER_TILE;
2271         if (y_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - s->sps->ctb_width])
2272             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2273     } else {
2274         if (!ctb_addr_in_slice > 0)
2275             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2276         if (ctb_addr_in_slice < s->sps->ctb_width)
2277             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2278     }
2279
2280     lc->ctb_left_flag = ((x_ctb > 0) && (ctb_addr_in_slice > 0) && !(lc->boundary_flags & BOUNDARY_LEFT_TILE));
2281     lc->ctb_up_flag   = ((y_ctb > 0) && (ctb_addr_in_slice >= s->sps->ctb_width) && !(lc->boundary_flags & BOUNDARY_UPPER_TILE));
2282     lc->ctb_up_right_flag = ((y_ctb > 0)  && (ctb_addr_in_slice+1 >= s->sps->ctb_width) && (s->pps->tile_id[ctb_addr_ts] == s->pps->tile_id[s->pps->ctb_addr_rs_to_ts[ctb_addr_rs+1 - s->sps->ctb_width]]));
2283     lc->ctb_up_left_flag = ((x_ctb > 0) && (y_ctb > 0)  && (ctb_addr_in_slice-1 >= s->sps->ctb_width) && (s->pps->tile_id[ctb_addr_ts] == s->pps->tile_id[s->pps->ctb_addr_rs_to_ts[ctb_addr_rs-1 - s->sps->ctb_width]]));
2284 }
2285
2286 static int hls_slice_data(HEVCContext *s)
2287 {
2288     int ctb_size    = 1 << s->sps->log2_ctb_size;
2289     int more_data   = 1;
2290     int x_ctb       = 0;
2291     int y_ctb       = 0;
2292     int ctb_addr_ts = s->pps->ctb_addr_rs_to_ts[s->sh.slice_ctb_addr_rs];
2293     int ret;
2294
2295     while (more_data && ctb_addr_ts < s->sps->ctb_size) {
2296         int ctb_addr_rs = s->pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2297
2298         x_ctb = (ctb_addr_rs % ((s->sps->width + ctb_size - 1) >> s->sps->log2_ctb_size)) << s->sps->log2_ctb_size;
2299         y_ctb = (ctb_addr_rs / ((s->sps->width + ctb_size - 1) >> s->sps->log2_ctb_size)) << s->sps->log2_ctb_size;
2300         hls_decode_neighbour(s, x_ctb, y_ctb, ctb_addr_ts);
2301
2302         ff_hevc_cabac_init(s, ctb_addr_ts);
2303
2304         hls_sao_param(s, x_ctb >> s->sps->log2_ctb_size, y_ctb >> s->sps->log2_ctb_size);
2305
2306         s->deblock[ctb_addr_rs].beta_offset = s->sh.beta_offset;
2307         s->deblock[ctb_addr_rs].tc_offset   = s->sh.tc_offset;
2308         s->filter_slice_edges[ctb_addr_rs]  = s->sh.slice_loop_filter_across_slices_enabled_flag;
2309
2310         ret = hls_coding_quadtree(s, x_ctb, y_ctb, s->sps->log2_ctb_size, 0);
2311         if (ret < 0)
2312             return ret;
2313         more_data = !ff_hevc_end_of_slice_flag_decode(s);
2314
2315         ctb_addr_ts++;
2316         ff_hevc_save_states(s, ctb_addr_ts);
2317         ff_hevc_hls_filters(s, x_ctb, y_ctb, ctb_size);
2318     }
2319
2320     if (x_ctb + ctb_size >= s->sps->width &&
2321         y_ctb + ctb_size >= s->sps->height)
2322         ff_hevc_hls_filter(s, x_ctb, y_ctb);
2323
2324     return ctb_addr_ts;
2325 }
2326
2327 /**
2328  * @return AVERROR_INVALIDDATA if the packet is not a valid NAL unit,
2329  * 0 if the unit should be skipped, 1 otherwise
2330  */
2331 static int hls_nal_unit(HEVCContext *s)
2332 {
2333     GetBitContext *gb = &s->HEVClc.gb;
2334     int nuh_layer_id;
2335
2336     if (get_bits1(gb) != 0)
2337         return AVERROR_INVALIDDATA;
2338
2339     s->nal_unit_type = get_bits(gb, 6);
2340
2341     nuh_layer_id   = get_bits(gb, 6);
2342     s->temporal_id = get_bits(gb, 3) - 1;
2343     if (s->temporal_id < 0)
2344         return AVERROR_INVALIDDATA;
2345
2346     av_log(s->avctx, AV_LOG_DEBUG,
2347            "nal_unit_type: %d, nuh_layer_id: %dtemporal_id: %d\n",
2348            s->nal_unit_type, nuh_layer_id, s->temporal_id);
2349
2350     return nuh_layer_id == 0;
2351 }
2352
2353 static void restore_tqb_pixels(HEVCContext *s)
2354 {
2355     int min_pu_size = 1 << s->sps->log2_min_pu_size;
2356     int x, y, c_idx;
2357
2358     for (c_idx = 0; c_idx < 3; c_idx++) {
2359         ptrdiff_t stride = s->frame->linesize[c_idx];
2360         int hshift       = s->sps->hshift[c_idx];
2361         int vshift       = s->sps->vshift[c_idx];
2362         for (y = 0; y < s->sps->min_pu_height; y++) {
2363             for (x = 0; x < s->sps->min_pu_width; x++) {
2364                 if (s->is_pcm[y * s->sps->min_pu_width + x]) {
2365                     int n;
2366                     int len      = min_pu_size >> hshift;
2367                     uint8_t *src = &s->frame->data[c_idx][((y << s->sps->log2_min_pu_size) >> vshift) * stride + (((x << s->sps->log2_min_pu_size) >> hshift) << s->sps->pixel_shift)];
2368                     uint8_t *dst = &s->sao_frame->data[c_idx][((y << s->sps->log2_min_pu_size) >> vshift) * stride + (((x << s->sps->log2_min_pu_size) >> hshift) << s->sps->pixel_shift)];
2369                     for (n = 0; n < (min_pu_size >> vshift); n++) {
2370                         memcpy(dst, src, len);
2371                         src += stride;
2372                         dst += stride;
2373                     }
2374                 }
2375             }
2376         }
2377     }
2378 }
2379
2380 static int set_side_data(HEVCContext *s)
2381 {
2382     AVFrame *out = s->ref->frame;
2383
2384     if (s->sei_frame_packing_present &&
2385         s->frame_packing_arrangement_type >= 3 &&
2386         s->frame_packing_arrangement_type <= 5 &&
2387         s->content_interpretation_type > 0 &&
2388         s->content_interpretation_type < 3) {
2389         AVStereo3D *stereo = av_stereo3d_create_side_data(out);
2390         if (!stereo)
2391             return AVERROR(ENOMEM);
2392
2393         switch (s->frame_packing_arrangement_type) {
2394         case 3:
2395             if (s->quincunx_subsampling)
2396                 stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
2397             else
2398                 stereo->type = AV_STEREO3D_SIDEBYSIDE;
2399             break;
2400         case 4:
2401             stereo->type = AV_STEREO3D_TOPBOTTOM;
2402             break;
2403         case 5:
2404             stereo->type = AV_STEREO3D_FRAMESEQUENCE;
2405             break;
2406         }
2407
2408         if (s->content_interpretation_type == 2)
2409             stereo->flags = AV_STEREO3D_FLAG_INVERT;
2410     }
2411
2412     if (s->sei_display_orientation_present &&
2413         (s->sei_anticlockwise_rotation || s->sei_hflip || s->sei_vflip)) {
2414         double angle = s->sei_anticlockwise_rotation * 360 / (double) (1 << 16);
2415         AVFrameSideData *rotation = av_frame_new_side_data(out,
2416                                                            AV_FRAME_DATA_DISPLAYMATRIX,
2417                                                            sizeof(int32_t) * 9);
2418         if (!rotation)
2419             return AVERROR(ENOMEM);
2420
2421         av_display_rotation_set((int32_t *)rotation->data, angle);
2422         av_display_matrix_flip((int32_t *)rotation->data,
2423                                s->sei_hflip, s->sei_vflip);
2424     }
2425
2426     return 0;
2427 }
2428
2429 static int hevc_frame_start(HEVCContext *s)
2430 {
2431     HEVCLocalContext *lc = &s->HEVClc;
2432     int ret;
2433
2434     memset(s->horizontal_bs, 0, 2 * s->bs_width * (s->bs_height + 1));
2435     memset(s->vertical_bs,   0, 2 * s->bs_width * (s->bs_height + 1));
2436     memset(s->cbf_luma,      0, s->sps->min_tb_width * s->sps->min_tb_height);
2437     memset(s->is_pcm,        0, s->sps->min_pu_width * s->sps->min_pu_height);
2438
2439     lc->start_of_tiles_x = 0;
2440     s->is_decoded        = 0;
2441     s->first_nal_type    = s->nal_unit_type;
2442
2443     if (s->pps->tiles_enabled_flag)
2444         lc->end_of_tiles_x = s->pps->column_width[0] << s->sps->log2_ctb_size;
2445
2446     ret = ff_hevc_set_new_ref(s, s->sps->sao_enabled ? &s->sao_frame : &s->frame,
2447                               s->poc);
2448     if (ret < 0)
2449         goto fail;
2450
2451     ret = ff_hevc_frame_rps(s);
2452     if (ret < 0) {
2453         av_log(s->avctx, AV_LOG_ERROR, "Error constructing the frame RPS.\n");
2454         goto fail;
2455     }
2456
2457     s->ref->frame->key_frame = IS_IRAP(s);
2458
2459     ret = set_side_data(s);
2460     if (ret < 0)
2461         goto fail;
2462
2463     av_frame_unref(s->output_frame);
2464     ret = ff_hevc_output_frame(s, s->output_frame, 0);
2465     if (ret < 0)
2466         goto fail;
2467
2468     ff_thread_finish_setup(s->avctx);
2469
2470     return 0;
2471
2472 fail:
2473     if (s->ref)
2474         ff_hevc_unref_frame(s, s->ref, ~0);
2475     s->ref = NULL;
2476     return ret;
2477 }
2478
2479 static int decode_nal_unit(HEVCContext *s, const HEVCNAL *nal)
2480 {
2481     HEVCLocalContext *lc = &s->HEVClc;
2482     GetBitContext *gb    = &lc->gb;
2483     int ctb_addr_ts, ret;
2484
2485     ret = init_get_bits8(gb, nal->data, nal->size);
2486     if (ret < 0)
2487         return ret;
2488
2489     ret = hls_nal_unit(s);
2490     if (ret < 0) {
2491         av_log(s->avctx, AV_LOG_ERROR, "Invalid NAL unit %d, skipping.\n",
2492                s->nal_unit_type);
2493         goto fail;
2494     } else if (!ret)
2495         return 0;
2496
2497     switch (s->nal_unit_type) {
2498     case NAL_VPS:
2499         ret = ff_hevc_decode_nal_vps(s);
2500         if (ret < 0)
2501             goto fail;
2502         break;
2503     case NAL_SPS:
2504         ret = ff_hevc_decode_nal_sps(s);
2505         if (ret < 0)
2506             goto fail;
2507         break;
2508     case NAL_PPS:
2509         ret = ff_hevc_decode_nal_pps(s);
2510         if (ret < 0)
2511             goto fail;
2512         break;
2513     case NAL_SEI_PREFIX:
2514     case NAL_SEI_SUFFIX:
2515         ret = ff_hevc_decode_nal_sei(s);
2516         if (ret < 0)
2517             goto fail;
2518         break;
2519     case NAL_TRAIL_R:
2520     case NAL_TRAIL_N:
2521     case NAL_TSA_N:
2522     case NAL_TSA_R:
2523     case NAL_STSA_N:
2524     case NAL_STSA_R:
2525     case NAL_BLA_W_LP:
2526     case NAL_BLA_W_RADL:
2527     case NAL_BLA_N_LP:
2528     case NAL_IDR_W_RADL:
2529     case NAL_IDR_N_LP:
2530     case NAL_CRA_NUT:
2531     case NAL_RADL_N:
2532     case NAL_RADL_R:
2533     case NAL_RASL_N:
2534     case NAL_RASL_R:
2535         ret = hls_slice_header(s);
2536         if (ret < 0)
2537             return ret;
2538
2539         if (s->max_ra == INT_MAX) {
2540             if (s->nal_unit_type == NAL_CRA_NUT || IS_BLA(s)) {
2541                 s->max_ra = s->poc;
2542             } else {
2543                 if (IS_IDR(s))
2544                     s->max_ra = INT_MIN;
2545             }
2546         }
2547
2548         if ((s->nal_unit_type == NAL_RASL_R || s->nal_unit_type == NAL_RASL_N) &&
2549             s->poc <= s->max_ra) {
2550             s->is_decoded = 0;
2551             break;
2552         } else {
2553             if (s->nal_unit_type == NAL_RASL_R && s->poc > s->max_ra)
2554                 s->max_ra = INT_MIN;
2555         }
2556
2557         if (s->sh.first_slice_in_pic_flag) {
2558             ret = hevc_frame_start(s);
2559             if (ret < 0)
2560                 return ret;
2561         } else if (!s->ref) {
2562             av_log(s->avctx, AV_LOG_ERROR, "First slice in a frame missing.\n");
2563             goto fail;
2564         }
2565
2566         if (s->nal_unit_type != s->first_nal_type) {
2567             av_log(s->avctx, AV_LOG_ERROR,
2568                    "Non-matching NAL types of the VCL NALUs: %d %d\n",
2569                    s->first_nal_type, s->nal_unit_type);
2570             return AVERROR_INVALIDDATA;
2571         }
2572
2573         if (!s->sh.dependent_slice_segment_flag &&
2574             s->sh.slice_type != I_SLICE) {
2575             ret = ff_hevc_slice_rpl(s);
2576             if (ret < 0) {
2577                 av_log(s->avctx, AV_LOG_WARNING,
2578                        "Error constructing the reference lists for the current slice.\n");
2579                 goto fail;
2580             }
2581         }
2582
2583         if (s->sh.first_slice_in_pic_flag && s->avctx->hwaccel) {
2584             ret = s->avctx->hwaccel->start_frame(s->avctx, NULL, 0);
2585             if (ret < 0)
2586                 goto fail;
2587         }
2588
2589         if (s->avctx->hwaccel) {
2590             ret = s->avctx->hwaccel->decode_slice(s->avctx, nal->raw_data, nal->raw_size);
2591             if (ret < 0)
2592                 goto fail;
2593         } else {
2594             ctb_addr_ts = hls_slice_data(s);
2595             if (ctb_addr_ts >= (s->sps->ctb_width * s->sps->ctb_height)) {
2596                 s->is_decoded = 1;
2597                 if ((s->pps->transquant_bypass_enable_flag ||
2598                      (s->sps->pcm.loop_filter_disable_flag && s->sps->pcm_enabled_flag)) &&
2599                     s->sps->sao_enabled)
2600                     restore_tqb_pixels(s);
2601             }
2602
2603             if (ctb_addr_ts < 0) {
2604                 ret = ctb_addr_ts;
2605                 goto fail;
2606             }
2607         }
2608         break;
2609     case NAL_EOS_NUT:
2610     case NAL_EOB_NUT:
2611         s->seq_decode = (s->seq_decode + 1) & 0xff;
2612         s->max_ra     = INT_MAX;
2613         break;
2614     case NAL_AUD:
2615     case NAL_FD_NUT:
2616         break;
2617     default:
2618         av_log(s->avctx, AV_LOG_INFO,
2619                "Skipping NAL unit %d\n", s->nal_unit_type);
2620     }
2621
2622     return 0;
2623 fail:
2624     if (s->avctx->err_recognition & AV_EF_EXPLODE)
2625         return ret;
2626     return 0;
2627 }
2628
2629 /* FIXME: This is adapted from ff_h264_decode_nal, avoiding duplication
2630  * between these functions would be nice. */
2631 static int extract_rbsp(const uint8_t *src, int length,
2632                         HEVCNAL *nal)
2633 {
2634     int i, si, di;
2635     uint8_t *dst;
2636
2637 #define STARTCODE_TEST                                                  \
2638         if (i + 2 < length && src[i + 1] == 0 && src[i + 2] <= 3) {     \
2639             if (src[i + 2] != 3) {                                      \
2640                 /* startcode, so we must be past the end */             \
2641                 length = i;                                             \
2642             }                                                           \
2643             break;                                                      \
2644         }
2645 #if HAVE_FAST_UNALIGNED
2646 #define FIND_FIRST_ZERO                                                 \
2647         if (i > 0 && !src[i])                                           \
2648             i--;                                                        \
2649         while (src[i])                                                  \
2650             i++
2651 #if HAVE_FAST_64BIT
2652     for (i = 0; i + 1 < length; i += 9) {
2653         if (!((~AV_RN64A(src + i) &
2654                (AV_RN64A(src + i) - 0x0100010001000101ULL)) &
2655               0x8000800080008080ULL))
2656             continue;
2657         FIND_FIRST_ZERO;
2658         STARTCODE_TEST;
2659         i -= 7;
2660     }
2661 #else
2662     for (i = 0; i + 1 < length; i += 5) {
2663         if (!((~AV_RN32A(src + i) &
2664                (AV_RN32A(src + i) - 0x01000101U)) &
2665               0x80008080U))
2666             continue;
2667         FIND_FIRST_ZERO;
2668         STARTCODE_TEST;
2669         i -= 3;
2670     }
2671 #endif /* HAVE_FAST_64BIT */
2672 #else
2673     for (i = 0; i + 1 < length; i += 2) {
2674         if (src[i])
2675             continue;
2676         if (i > 0 && src[i - 1] == 0)
2677             i--;
2678         STARTCODE_TEST;
2679     }
2680 #endif /* HAVE_FAST_UNALIGNED */
2681
2682     if (i >= length - 1) { // no escaped 0
2683         nal->data     =
2684         nal->raw_data = src;
2685         nal->size     =
2686         nal->raw_size = length;
2687         return length;
2688     }
2689
2690     av_fast_malloc(&nal->rbsp_buffer, &nal->rbsp_buffer_size,
2691                    length + FF_INPUT_BUFFER_PADDING_SIZE);
2692     if (!nal->rbsp_buffer)
2693         return AVERROR(ENOMEM);
2694
2695     dst = nal->rbsp_buffer;
2696
2697     memcpy(dst, src, i);
2698     si = di = i;
2699     while (si + 2 < length) {
2700         // remove escapes (very rare 1:2^22)
2701         if (src[si + 2] > 3) {
2702             dst[di++] = src[si++];
2703             dst[di++] = src[si++];
2704         } else if (src[si] == 0 && src[si + 1] == 0) {
2705             if (src[si + 2] == 3) { // escape
2706                 dst[di++] = 0;
2707                 dst[di++] = 0;
2708                 si       += 3;
2709
2710                 continue;
2711             } else // next start code
2712                 goto nsc;
2713         }
2714
2715         dst[di++] = src[si++];
2716     }
2717     while (si < length)
2718         dst[di++] = src[si++];
2719
2720 nsc:
2721     memset(dst + di, 0, FF_INPUT_BUFFER_PADDING_SIZE);
2722
2723     nal->data = dst;
2724     nal->size = di;
2725     nal->raw_data = src;
2726     nal->raw_size = si;
2727     return si;
2728 }
2729
2730 static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length)
2731 {
2732     int i, consumed, ret = 0;
2733
2734     s->ref = NULL;
2735     s->eos = 0;
2736
2737     /* split the input packet into NAL units, so we know the upper bound on the
2738      * number of slices in the frame */
2739     s->nb_nals = 0;
2740     while (length >= 4) {
2741         HEVCNAL *nal;
2742         int extract_length = 0;
2743
2744         if (s->is_nalff) {
2745             int i;
2746             for (i = 0; i < s->nal_length_size; i++)
2747                 extract_length = (extract_length << 8) | buf[i];
2748             buf    += s->nal_length_size;
2749             length -= s->nal_length_size;
2750
2751             if (extract_length > length) {
2752                 av_log(s->avctx, AV_LOG_ERROR, "Invalid NAL unit size.\n");
2753                 ret = AVERROR_INVALIDDATA;
2754                 goto fail;
2755             }
2756         } else {
2757             if (buf[2] == 0) {
2758                 length--;
2759                 buf++;
2760                 continue;
2761             }
2762             if (buf[0] != 0 || buf[1] != 0 || buf[2] != 1) {
2763                 ret = AVERROR_INVALIDDATA;
2764                 goto fail;
2765             }
2766
2767             buf           += 3;
2768             length        -= 3;
2769             extract_length = length;
2770         }
2771
2772         if (s->nals_allocated < s->nb_nals + 1) {
2773             int new_size = s->nals_allocated + 1;
2774             HEVCNAL *tmp = av_realloc_array(s->nals, new_size, sizeof(*tmp));
2775             if (!tmp) {
2776                 ret = AVERROR(ENOMEM);
2777                 goto fail;
2778             }
2779             s->nals = tmp;
2780             memset(s->nals + s->nals_allocated, 0,
2781                    (new_size - s->nals_allocated) * sizeof(*tmp));
2782             s->nals_allocated = new_size;
2783         }
2784         nal = &s->nals[s->nb_nals++];
2785
2786         consumed = extract_rbsp(buf, extract_length, nal);
2787         if (consumed < 0) {
2788             ret = consumed;
2789             goto fail;
2790         }
2791
2792         ret = init_get_bits8(&s->HEVClc.gb, nal->data, nal->size);
2793         if (ret < 0)
2794             goto fail;
2795         hls_nal_unit(s);
2796
2797         if (s->nal_unit_type == NAL_EOB_NUT ||
2798             s->nal_unit_type == NAL_EOS_NUT)
2799             s->eos = 1;
2800
2801         buf    += consumed;
2802         length -= consumed;
2803     }
2804
2805     /* parse the NAL units */
2806     for (i = 0; i < s->nb_nals; i++) {
2807         int ret = decode_nal_unit(s, &s->nals[i]);
2808         if (ret < 0) {
2809             av_log(s->avctx, AV_LOG_WARNING,
2810                    "Error parsing NAL unit #%d.\n", i);
2811             goto fail;
2812         }
2813     }
2814
2815 fail:
2816     if (s->ref)
2817         ff_thread_report_progress(&s->ref->tf, INT_MAX, 0);
2818
2819     return ret;
2820 }
2821
2822 static void print_md5(void *log_ctx, int level, uint8_t md5[16])
2823 {
2824     int i;
2825     for (i = 0; i < 16; i++)
2826         av_log(log_ctx, level, "%02"PRIx8, md5[i]);
2827 }
2828
2829 static int verify_md5(HEVCContext *s, AVFrame *frame)
2830 {
2831     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
2832     int pixel_shift;
2833     int i, j;
2834
2835     if (!desc)
2836         return AVERROR(EINVAL);
2837
2838     pixel_shift = desc->comp[0].depth_minus1 > 7;
2839
2840     av_log(s->avctx, AV_LOG_DEBUG, "Verifying checksum for frame with POC %d: ",
2841            s->poc);
2842
2843     /* the checksums are LE, so we have to byteswap for >8bpp formats
2844      * on BE arches */
2845 #if HAVE_BIGENDIAN
2846     if (pixel_shift && !s->checksum_buf) {
2847         av_fast_malloc(&s->checksum_buf, &s->checksum_buf_size,
2848                        FFMAX3(frame->linesize[0], frame->linesize[1],
2849                               frame->linesize[2]));
2850         if (!s->checksum_buf)
2851             return AVERROR(ENOMEM);
2852     }
2853 #endif
2854
2855     for (i = 0; frame->data[i]; i++) {
2856         int width  = s->avctx->coded_width;
2857         int height = s->avctx->coded_height;
2858         int w = (i == 1 || i == 2) ? (width  >> desc->log2_chroma_w) : width;
2859         int h = (i == 1 || i == 2) ? (height >> desc->log2_chroma_h) : height;
2860         uint8_t md5[16];
2861
2862         av_md5_init(s->md5_ctx);
2863         for (j = 0; j < h; j++) {
2864             const uint8_t *src = frame->data[i] + j * frame->linesize[i];
2865 #if HAVE_BIGENDIAN
2866             if (pixel_shift) {
2867                 s->bdsp.bswap16_buf((uint16_t *) s->checksum_buf,
2868                                     (const uint16_t *) src, w);
2869                 src = s->checksum_buf;
2870             }
2871 #endif
2872             av_md5_update(s->md5_ctx, src, w << pixel_shift);
2873         }
2874         av_md5_final(s->md5_ctx, md5);
2875
2876         if (!memcmp(md5, s->md5[i], 16)) {
2877             av_log   (s->avctx, AV_LOG_DEBUG, "plane %d - correct ", i);
2878             print_md5(s->avctx, AV_LOG_DEBUG, md5);
2879             av_log   (s->avctx, AV_LOG_DEBUG, "; ");
2880         } else {
2881             av_log   (s->avctx, AV_LOG_ERROR, "mismatching checksum of plane %d - ", i);
2882             print_md5(s->avctx, AV_LOG_ERROR, md5);
2883             av_log   (s->avctx, AV_LOG_ERROR, " != ");
2884             print_md5(s->avctx, AV_LOG_ERROR, s->md5[i]);
2885             av_log   (s->avctx, AV_LOG_ERROR, "\n");
2886             return AVERROR_INVALIDDATA;
2887         }
2888     }
2889
2890     av_log(s->avctx, AV_LOG_DEBUG, "\n");
2891
2892     return 0;
2893 }
2894
2895 static int hevc_decode_frame(AVCodecContext *avctx, void *data, int *got_output,
2896                              AVPacket *avpkt)
2897 {
2898     int ret;
2899     HEVCContext *s = avctx->priv_data;
2900
2901     if (!avpkt->size) {
2902         ret = ff_hevc_output_frame(s, data, 1);
2903         if (ret < 0)
2904             return ret;
2905
2906         *got_output = ret;
2907         return 0;
2908     }
2909
2910     s->ref = NULL;
2911     ret    = decode_nal_units(s, avpkt->data, avpkt->size);
2912     if (ret < 0)
2913         return ret;
2914
2915     if (avctx->hwaccel) {
2916         if (s->ref && avctx->hwaccel->end_frame(avctx) < 0)
2917             av_log(avctx, AV_LOG_ERROR,
2918                    "hardware accelerator failed to decode picture\n");
2919     } else {
2920         /* verify the SEI checksum */
2921         if (avctx->err_recognition & AV_EF_CRCCHECK && s->is_decoded &&
2922             s->is_md5) {
2923             ret = verify_md5(s, s->ref->frame);
2924             if (ret < 0 && avctx->err_recognition & AV_EF_EXPLODE) {
2925                 ff_hevc_unref_frame(s, s->ref, ~0);
2926                 return ret;
2927             }
2928         }
2929     }
2930     s->is_md5 = 0;
2931
2932     if (s->is_decoded) {
2933         av_log(avctx, AV_LOG_DEBUG, "Decoded frame with POC %d.\n", s->poc);
2934         s->is_decoded = 0;
2935     }
2936
2937     if (s->output_frame->buf[0]) {
2938         av_frame_move_ref(data, s->output_frame);
2939         *got_output = 1;
2940     }
2941
2942     return avpkt->size;
2943 }
2944
2945 static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
2946 {
2947     int ret = ff_thread_ref_frame(&dst->tf, &src->tf);
2948     if (ret < 0)
2949         return ret;
2950
2951     dst->tab_mvf_buf = av_buffer_ref(src->tab_mvf_buf);
2952     if (!dst->tab_mvf_buf)
2953         goto fail;
2954     dst->tab_mvf = src->tab_mvf;
2955
2956     dst->rpl_tab_buf = av_buffer_ref(src->rpl_tab_buf);
2957     if (!dst->rpl_tab_buf)
2958         goto fail;
2959     dst->rpl_tab = src->rpl_tab;
2960
2961     dst->rpl_buf = av_buffer_ref(src->rpl_buf);
2962     if (!dst->rpl_buf)
2963         goto fail;
2964
2965     dst->poc        = src->poc;
2966     dst->ctb_count  = src->ctb_count;
2967     dst->window     = src->window;
2968     dst->flags      = src->flags;
2969     dst->sequence   = src->sequence;
2970
2971     if (src->hwaccel_picture_private) {
2972         dst->hwaccel_priv_buf = av_buffer_ref(src->hwaccel_priv_buf);
2973         if (!dst->hwaccel_priv_buf)
2974             goto fail;
2975         dst->hwaccel_picture_private = dst->hwaccel_priv_buf->data;
2976     }
2977
2978     return 0;
2979 fail:
2980     ff_hevc_unref_frame(s, dst, ~0);
2981     return AVERROR(ENOMEM);
2982 }
2983
2984 static av_cold int hevc_decode_free(AVCodecContext *avctx)
2985 {
2986     HEVCContext       *s = avctx->priv_data;
2987     int i;
2988
2989     pic_arrays_free(s);
2990
2991     av_freep(&s->md5_ctx);
2992
2993     av_frame_free(&s->tmp_frame);
2994     av_frame_free(&s->output_frame);
2995
2996     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2997         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2998         av_frame_free(&s->DPB[i].frame);
2999     }
3000
3001     for (i = 0; i < FF_ARRAY_ELEMS(s->vps_list); i++)
3002         av_buffer_unref(&s->vps_list[i]);
3003     for (i = 0; i < FF_ARRAY_ELEMS(s->sps_list); i++)
3004         av_buffer_unref(&s->sps_list[i]);
3005     for (i = 0; i < FF_ARRAY_ELEMS(s->pps_list); i++)
3006         av_buffer_unref(&s->pps_list[i]);
3007
3008     for (i = 0; i < s->nals_allocated; i++)
3009         av_freep(&s->nals[i].rbsp_buffer);
3010     av_freep(&s->nals);
3011     s->nals_allocated = 0;
3012
3013     return 0;
3014 }
3015
3016 static av_cold int hevc_init_context(AVCodecContext *avctx)
3017 {
3018     HEVCContext *s = avctx->priv_data;
3019     int i;
3020
3021     s->avctx = avctx;
3022
3023     s->tmp_frame = av_frame_alloc();
3024     if (!s->tmp_frame)
3025         goto fail;
3026
3027     s->output_frame = av_frame_alloc();
3028     if (!s->output_frame)
3029         goto fail;
3030
3031     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
3032         s->DPB[i].frame = av_frame_alloc();
3033         if (!s->DPB[i].frame)
3034             goto fail;
3035         s->DPB[i].tf.f = s->DPB[i].frame;
3036     }
3037
3038     s->max_ra = INT_MAX;
3039
3040     s->md5_ctx = av_md5_alloc();
3041     if (!s->md5_ctx)
3042         goto fail;
3043
3044     ff_bswapdsp_init(&s->bdsp);
3045
3046     s->context_initialized = 1;
3047
3048     return 0;
3049
3050 fail:
3051     hevc_decode_free(avctx);
3052     return AVERROR(ENOMEM);
3053 }
3054
3055 static int hevc_update_thread_context(AVCodecContext *dst,
3056                                       const AVCodecContext *src)
3057 {
3058     HEVCContext *s  = dst->priv_data;
3059     HEVCContext *s0 = src->priv_data;
3060     int i, ret;
3061
3062     if (!s->context_initialized) {
3063         ret = hevc_init_context(dst);
3064         if (ret < 0)
3065             return ret;
3066     }
3067
3068     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
3069         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
3070         if (s0->DPB[i].frame->buf[0]) {
3071             ret = hevc_ref_frame(s, &s->DPB[i], &s0->DPB[i]);
3072             if (ret < 0)
3073                 return ret;
3074         }
3075     }
3076
3077     for (i = 0; i < FF_ARRAY_ELEMS(s->vps_list); i++) {
3078         av_buffer_unref(&s->vps_list[i]);
3079         if (s0->vps_list[i]) {
3080             s->vps_list[i] = av_buffer_ref(s0->vps_list[i]);
3081             if (!s->vps_list[i])
3082                 return AVERROR(ENOMEM);
3083         }
3084     }
3085
3086     for (i = 0; i < FF_ARRAY_ELEMS(s->sps_list); i++) {
3087         av_buffer_unref(&s->sps_list[i]);
3088         if (s0->sps_list[i]) {
3089             s->sps_list[i] = av_buffer_ref(s0->sps_list[i]);
3090             if (!s->sps_list[i])
3091                 return AVERROR(ENOMEM);
3092         }
3093     }
3094
3095     for (i = 0; i < FF_ARRAY_ELEMS(s->pps_list); i++) {
3096         av_buffer_unref(&s->pps_list[i]);
3097         if (s0->pps_list[i]) {
3098             s->pps_list[i] = av_buffer_ref(s0->pps_list[i]);
3099             if (!s->pps_list[i])
3100                 return AVERROR(ENOMEM);
3101         }
3102     }
3103
3104     if (s->sps != s0->sps)
3105         ret = set_sps(s, s0->sps);
3106
3107     s->seq_decode = s0->seq_decode;
3108     s->seq_output = s0->seq_output;
3109     s->pocTid0    = s0->pocTid0;
3110     s->max_ra     = s0->max_ra;
3111
3112     s->is_nalff        = s0->is_nalff;
3113     s->nal_length_size = s0->nal_length_size;
3114
3115     if (s0->eos) {
3116         s->seq_decode = (s->seq_decode + 1) & 0xff;
3117         s->max_ra = INT_MAX;
3118     }
3119
3120     return 0;
3121 }
3122
3123 static int hevc_decode_extradata(HEVCContext *s)
3124 {
3125     AVCodecContext *avctx = s->avctx;
3126     GetByteContext gb;
3127     int ret;
3128
3129     bytestream2_init(&gb, avctx->extradata, avctx->extradata_size);
3130
3131     if (avctx->extradata_size > 3 &&
3132         (avctx->extradata[0] || avctx->extradata[1] ||
3133          avctx->extradata[2] > 1)) {
3134         /* It seems the extradata is encoded as hvcC format.
3135          * Temporarily, we support configurationVersion==0 until 14496-15 3rd
3136          * is finalized. When finalized, configurationVersion will be 1 and we
3137          * can recognize hvcC by checking if avctx->extradata[0]==1 or not. */
3138         int i, j, num_arrays, nal_len_size;
3139
3140         s->is_nalff = 1;
3141
3142         bytestream2_skip(&gb, 21);
3143         nal_len_size = (bytestream2_get_byte(&gb) & 3) + 1;
3144         num_arrays   = bytestream2_get_byte(&gb);
3145
3146         /* nal units in the hvcC always have length coded with 2 bytes,
3147          * so put a fake nal_length_size = 2 while parsing them */
3148         s->nal_length_size = 2;
3149
3150         /* Decode nal units from hvcC. */
3151         for (i = 0; i < num_arrays; i++) {
3152             int type = bytestream2_get_byte(&gb) & 0x3f;
3153             int cnt  = bytestream2_get_be16(&gb);
3154
3155             for (j = 0; j < cnt; j++) {
3156                 // +2 for the nal size field
3157                 int nalsize = bytestream2_peek_be16(&gb) + 2;
3158                 if (bytestream2_get_bytes_left(&gb) < nalsize) {
3159                     av_log(s->avctx, AV_LOG_ERROR,
3160                            "Invalid NAL unit size in extradata.\n");
3161                     return AVERROR_INVALIDDATA;
3162                 }
3163
3164                 ret = decode_nal_units(s, gb.buffer, nalsize);
3165                 if (ret < 0) {
3166                     av_log(avctx, AV_LOG_ERROR,
3167                            "Decoding nal unit %d %d from hvcC failed\n",
3168                            type, i);
3169                     return ret;
3170                 }
3171                 bytestream2_skip(&gb, nalsize);
3172             }
3173         }
3174
3175         /* Now store right nal length size, that will be used to parse
3176          * all other nals */
3177         s->nal_length_size = nal_len_size;
3178     } else {
3179         s->is_nalff = 0;
3180         ret = decode_nal_units(s, avctx->extradata, avctx->extradata_size);
3181         if (ret < 0)
3182             return ret;
3183     }
3184     return 0;
3185 }
3186
3187 static av_cold int hevc_decode_init(AVCodecContext *avctx)
3188 {
3189     HEVCContext *s = avctx->priv_data;
3190     int ret;
3191
3192     ff_init_cabac_states();
3193
3194     avctx->internal->allocate_progress = 1;
3195
3196     ret = hevc_init_context(avctx);
3197     if (ret < 0)
3198         return ret;
3199
3200     if (avctx->extradata_size > 0 && avctx->extradata) {
3201         ret = hevc_decode_extradata(s);
3202         if (ret < 0) {
3203             hevc_decode_free(avctx);
3204             return ret;
3205         }
3206     }
3207
3208     return 0;
3209 }
3210
3211 static av_cold int hevc_init_thread_copy(AVCodecContext *avctx)
3212 {
3213     HEVCContext *s = avctx->priv_data;
3214     int ret;
3215
3216     memset(s, 0, sizeof(*s));
3217
3218     ret = hevc_init_context(avctx);
3219     if (ret < 0)
3220         return ret;
3221
3222     return 0;
3223 }
3224
3225 static void hevc_decode_flush(AVCodecContext *avctx)
3226 {
3227     HEVCContext *s = avctx->priv_data;
3228     ff_hevc_flush_dpb(s);
3229     s->max_ra = INT_MAX;
3230 }
3231
3232 #define OFFSET(x) offsetof(HEVCContext, x)
3233 #define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
3234
3235 static const AVProfile profiles[] = {
3236     { FF_PROFILE_HEVC_MAIN,                 "Main"                },
3237     { FF_PROFILE_HEVC_MAIN_10,              "Main 10"             },
3238     { FF_PROFILE_HEVC_MAIN_STILL_PICTURE,   "Main Still Picture"  },
3239     { FF_PROFILE_UNKNOWN },
3240 };
3241
3242 static const AVOption options[] = {
3243     { "apply_defdispwin", "Apply default display window from VUI", OFFSET(apply_defdispwin),
3244         AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, PAR },
3245     { NULL },
3246 };
3247
3248 static const AVClass hevc_decoder_class = {
3249     .class_name = "HEVC decoder",
3250     .item_name  = av_default_item_name,
3251     .option     = options,
3252     .version    = LIBAVUTIL_VERSION_INT,
3253 };
3254
3255 AVCodec ff_hevc_decoder = {
3256     .name                  = "hevc",
3257     .long_name             = NULL_IF_CONFIG_SMALL("HEVC (High Efficiency Video Coding)"),
3258     .type                  = AVMEDIA_TYPE_VIDEO,
3259     .id                    = AV_CODEC_ID_HEVC,
3260     .priv_data_size        = sizeof(HEVCContext),
3261     .priv_class            = &hevc_decoder_class,
3262     .init                  = hevc_decode_init,
3263     .close                 = hevc_decode_free,
3264     .decode                = hevc_decode_frame,
3265     .flush                 = hevc_decode_flush,
3266     .update_thread_context = hevc_update_thread_context,
3267     .init_thread_copy      = hevc_init_thread_copy,
3268     .capabilities          = CODEC_CAP_DR1 | CODEC_CAP_DELAY |
3269                              CODEC_CAP_FRAME_THREADS,
3270     .profiles              = NULL_IF_CONFIG_SMALL(profiles),
3271 };