a7571e2b5f43985d0ba51cc398519554623d437a
[vlc.git] / modules / codec / hxxx_helper.c
1 /*****************************************************************************
2  * hxxx_helper.c: AnnexB / avcC helper for dumb decoders
3  *****************************************************************************
4  * Copyright (C) 2017 VLC authors and VideoLAN
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by
8  * the Free Software Foundation; either version 2.1 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
19  *****************************************************************************/
20
21 #ifdef HAVE_CONFIG_H
22 # include "config.h"
23 #endif
24
25 #include <stdint.h>
26 #include <assert.h>
27
28 #include <vlc_common.h>
29 #include <vlc_bits.h>
30
31 #include "hxxx_helper.h"
32 #include "../packetizer/hxxx_nal.h"
33 #include "../packetizer/h264_slice.h"
34
35 void
36 hxxx_helper_init(struct hxxx_helper *hh, vlc_object_t *p_obj,
37                  vlc_fourcc_t i_codec, bool b_need_xvcC)
38 {
39     assert(i_codec == VLC_CODEC_H264 || i_codec == VLC_CODEC_HEVC);
40
41     memset(hh, 0, sizeof(struct hxxx_helper));
42     hh->p_obj = p_obj;
43     hh->i_codec = i_codec;
44     switch (i_codec)
45     {
46         case VLC_CODEC_H264:
47             break;
48     }
49     hh->b_need_xvcC = b_need_xvcC;
50 }
51
52 void
53 hxxx_helper_clean(struct hxxx_helper *hh)
54 {
55     switch (hh->i_codec)
56     {
57         case VLC_CODEC_H264:
58             for (size_t i = 0; i <= H264_SPS_ID_MAX; ++i)
59             {
60                 struct hxxx_helper_nal *hnal = &hh->h264.sps_list[i];
61                 if (hnal->b)
62                 {
63                     block_Release(hnal->b);
64                     h264_release_sps(hnal->h264_sps);
65                 }
66             }
67             for (size_t i = 0; i <= H264_PPS_ID_MAX; ++i)
68             {
69                 struct hxxx_helper_nal *hnal = &hh->h264.pps_list[i];
70                 if (hnal->b)
71                 {
72                     block_Release(hnal->b);
73                     h264_release_pps(hnal->h264_pps);
74                 }
75             }
76             break;
77         case VLC_CODEC_HEVC:
78             free(hh->hevc.p_annexb_config_nal);
79             break;
80         default:
81             vlc_assert_unreachable();
82     }
83 }
84
85 #define HELPER_FOREACH_NAL(it, p_nal_list, i_nal_count, i_nal_max) \
86     for (size_t ii = 0, i_nal_nb = 0; i < i_nal_max && i_nal_count > i_nal_nb; ++ii) \
87         if (p_nal_list[ii].b != NULL && (it = &p_nal_list[ii]) && ++i_nal_nb)
88
89 static int
90 helper_dup_buf(struct hxxx_helper_nal *p_nal,
91                const uint8_t *p_nal_buf, size_t i_nal_buf)
92 {
93     if (!p_nal->b)
94     {
95         p_nal->b = block_Alloc(i_nal_buf);
96         if (!p_nal->b)
97             return VLC_ENOMEM;
98     }
99     else if (p_nal->b != NULL && i_nal_buf > p_nal->b->i_buffer)
100     {
101         block_t *b = block_TryRealloc(p_nal->b, 0, i_nal_buf);
102         if (b == NULL)
103             return VLC_ENOMEM;
104         p_nal->b = b;
105     }
106     memcpy(p_nal->b->p_buffer, p_nal_buf, i_nal_buf);
107     p_nal->b->i_buffer = i_nal_buf;
108     return VLC_SUCCESS;
109 }
110
111 static inline const struct hxxx_helper_nal *
112 helper_search_nal(const struct hxxx_helper_nal *p_nal_list, size_t i_nal_count,
113                   size_t i_nal_max, const void *p_nal_buf, size_t i_nal_buf)
114 {
115     size_t i_nal_nb = 0;
116     for (size_t i = 0; i < i_nal_max && i_nal_count > i_nal_nb; ++i)
117     {
118         const struct hxxx_helper_nal *p_nal = &p_nal_list[i];
119         if (p_nal->b == NULL)
120             continue;
121         i_nal_nb++;
122         const int i_diff = i_nal_buf - p_nal->b->i_buffer;
123         if (i_diff == 0 && memcmp(p_nal_buf, p_nal->b->p_buffer, i_nal_buf) == 0)
124             return p_nal;
125     }
126     return NULL;
127 }
128 #define helper_search_sps(hh, p_nal, i_nal) \
129     helper_search_nal(hh->h264.sps_list, hh->h264.i_sps_count, \
130                       H264_SPS_ID_MAX+1, p_nal, i_nal)
131 #define helper_search_pps(hh, p_nal, i_nal) \
132     helper_search_nal(hh->h264.pps_list, hh->h264.i_pps_count, \
133                       H264_PPS_ID_MAX+1, p_nal, i_nal)
134
135 static inline bool
136 helper_nal_length_valid(struct hxxx_helper *hh)
137 {
138     return hh->i_nal_length_size == 1 || hh->i_nal_length_size == 2
139         || hh->i_nal_length_size == 4;
140 }
141
142 static int
143 h264_helper_parse_nal(struct hxxx_helper *hh, const uint8_t *p_buf, size_t i_buf,
144                       uint8_t i_nal_length_size, bool *p_config_changed)
145 {
146     const uint8_t *p_nal;
147     size_t i_nal;
148     hxxx_iterator_ctx_t it;
149     hxxx_iterator_init(&it, p_buf, i_buf, i_nal_length_size);
150     *p_config_changed = false;
151
152     while ((i_nal_length_size) ? hxxx_iterate_next(&it, &p_nal, &i_nal)
153                                : hxxx_annexb_iterate_next(&it, &p_nal, &i_nal))
154     {
155         if (i_nal < 2)
156             continue;
157
158         const enum h264_nal_unit_type_e i_nal_type = p_nal[0] & 0x1F;
159
160         if (i_nal_type == H264_NAL_SPS)
161         {
162             if (helper_search_sps(hh, p_nal, i_nal) != NULL)
163                 continue;
164             h264_sequence_parameter_set_t *p_sps =
165                 h264_decode_sps(p_nal, i_nal, true);
166             if (!p_sps)
167                 return VLC_EGENERIC;
168
169             struct hxxx_helper_nal *hnal = &hh->h264.sps_list[p_sps->i_id];
170             if (helper_dup_buf(hnal, p_nal, i_nal))
171             {
172                 h264_release_sps(p_sps);
173                 return VLC_EGENERIC;
174             }
175             if (hnal->h264_sps)
176                 h264_release_sps(hnal->h264_sps);
177             else
178                 hh->h264.i_sps_count++;
179
180             hnal->h264_sps = p_sps;
181             *p_config_changed = true;
182             hh->h264.i_current_sps = p_sps->i_id;
183             msg_Dbg(hh->p_obj, "new SPS parsed: %u\n", p_sps->i_id);
184         }
185         else if (i_nal_type == H264_NAL_PPS)
186         {
187             if (helper_search_pps(hh, p_nal, i_nal) != NULL)
188                 continue;
189             h264_picture_parameter_set_t *p_pps =
190                 h264_decode_pps(p_nal, i_nal, true);
191             if (!p_pps)
192                 return VLC_EGENERIC;
193
194             struct hxxx_helper_nal *hnal = &hh->h264.pps_list[p_pps->i_id];
195
196             if (helper_dup_buf(hnal, p_nal, i_nal))
197             {
198                 h264_release_pps(p_pps);
199                 return VLC_EGENERIC;
200             }
201             if (hnal->h264_pps)
202                 h264_release_pps(hnal->h264_pps);
203             else
204                 hh->h264.i_pps_count++;
205
206             hnal->h264_pps = p_pps;
207             *p_config_changed = true;
208             msg_Dbg(hh->p_obj, "new  PPS parsed: %u\n", p_pps->i_id);
209         }
210         else if (i_nal_type <= H264_NAL_SLICE_IDR
211               && i_nal_type != H264_NAL_UNKNOWN)
212         {
213             if (hh->h264.i_sps_count > 1)
214             {
215                 /* There is more than one SPS. Get the PPS id of the current
216                  * SLICE in order to get the current SPS id */
217
218                 /* Get the PPS id from the slice: inspirated from
219                  * h264_decode_slice() */
220                 bs_t s;
221                 bs_init(&s, p_nal, i_nal);
222                 bs_skip(&s, 8);
223                 bs_read_ue(&s);
224                 bs_read_ue(&s);
225                 unsigned i_pps_id = bs_read_ue(&s);
226                 if (i_pps_id > H264_PPS_ID_MAX)
227                     return VLC_EGENERIC;
228
229                 struct hxxx_helper_nal *hpps = &hh->h264.pps_list[i_pps_id];
230                 if (hpps->b == NULL)
231                     return VLC_EGENERIC;
232
233                 struct hxxx_helper_nal *hsps =
234                     &hh->h264.sps_list[hpps->h264_pps->i_sps_id];
235                 if (hsps->b == NULL)
236                     return VLC_EGENERIC;
237
238                 assert(hpps->h264_pps->i_sps_id == hsps->h264_sps->i_id);
239                 if (hsps->h264_sps->i_id != hh->h264.i_current_sps)
240                 {
241                     hh->h264.i_current_sps = hsps->h264_sps->i_id;
242                     *p_config_changed = true;
243                 }
244             }
245             break; /* No need to parse further NAL */
246         }
247     }
248     return VLC_SUCCESS;
249 }
250
251 static int
252 helper_process_avcC_h264(struct hxxx_helper *hh, const uint8_t *p_buf,
253                          size_t i_buf)
254 {
255     if (i_buf < H264_MIN_AVCC_SIZE)
256         return VLC_EGENERIC;
257
258     p_buf += 5; i_buf -= 5;
259
260     for (unsigned int j = 0; j < 2 && i_buf > 0; j++)
261     {
262         /* First time is SPS, Second is PPS */
263         const unsigned int i_num_nal = p_buf[0] & (j == 0 ? 0x1f : 0xff);
264         p_buf++; i_buf--;
265
266         for (unsigned int i = 0; i < i_num_nal && i_buf >= 2; i++)
267         {
268             uint16_t i_nal_size = (p_buf[0] << 8) | p_buf[1];
269             if (i_nal_size > i_buf - 2)
270                 return VLC_EGENERIC;
271             bool b_unused;
272             int i_ret = h264_helper_parse_nal(hh, p_buf, i_nal_size + 2, 2,
273                                               &b_unused);
274             if (i_ret != VLC_SUCCESS)
275                 return i_ret;
276             p_buf += i_nal_size + 2;
277             i_buf -= i_nal_size + 2;
278         }
279     }
280
281     return VLC_SUCCESS;
282 }
283
284 static int
285 h264_helper_set_extra(struct hxxx_helper *hh, const void *p_extra,
286                       size_t i_extra)
287 {
288     if (i_extra == 0)
289     {
290         /* AnnexB case */
291         hh->i_nal_length_size = 4;
292         return VLC_SUCCESS;
293     }
294     else if (h264_isavcC(p_extra, i_extra))
295     {
296         hh->i_nal_length_size = (((uint8_t*)p_extra)[4] & 0x03) + 1;
297         if (!helper_nal_length_valid(hh))
298             return VLC_EGENERIC;
299         hh->b_is_xvcC = true;
300
301         /* XXX h264_AVC_to_AnnexB() works only with a i_nal_length_size of 4.
302          * If nal_length_size is smaller than 4, fallback to SW decoding. I
303          * don't know if it's worth the effort to fix h264_AVC_to_AnnexB() for
304          * a smaller nal_length_size. Indeed, this case will happen only with
305          * very small resolutions, where hardware decoders are not that useful.
306          * -Thomas */
307         if (!hh->b_need_xvcC && hh->i_nal_length_size != 4)
308         {
309             msg_Dbg(hh->p_obj, "nal_length_size is too small");
310             return VLC_EGENERIC;
311         }
312
313         return helper_process_avcC_h264(hh, p_extra, i_extra);
314     }
315     else /* Can't handle extra that is not avcC */
316         return VLC_EGENERIC;
317 }
318
319 static int
320 hevc_helper_set_extra(struct hxxx_helper *hh, const void *p_extra,
321                       size_t i_extra)
322 {
323     if (i_extra == 0)
324     {
325         /* AnnexB case */
326         hh->i_nal_length_size = 4;
327         return VLC_SUCCESS;
328     }
329     else if (hevc_ishvcC(p_extra, i_extra))
330     {
331         hh->i_nal_length_size = hevc_getNALLengthSize(p_extra);
332         if (!helper_nal_length_valid(hh))
333             return VLC_EGENERIC;
334         hh->b_is_xvcC = true;
335
336         if (hh->b_need_xvcC)
337             return VLC_SUCCESS;
338
339         size_t i_buf;
340         uint8_t *p_buf = hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra, &i_buf,
341                                                  NULL);
342         if (!p_buf)
343         {
344             msg_Dbg(hh->p_obj, "hevc_hvcC_to_AnnexB_NAL failed");
345             return VLC_EGENERIC;
346         }
347
348         hh->hevc.p_annexb_config_nal = p_buf;
349         hh->hevc.i_annexb_config_nal = i_buf;
350         return VLC_SUCCESS;
351     }
352     else /* Can't handle extra that is not avcC */
353         return VLC_EGENERIC;
354 }
355
356 static block_t *
357 helper_process_block_xvcc2annexb(struct hxxx_helper *hh, block_t *p_block,
358                                  bool *p_config_changed)
359 {
360     assert(helper_nal_length_valid(hh));
361     if (p_config_changed != NULL)
362         *p_config_changed = false;
363     h264_AVC_to_AnnexB(p_block->p_buffer, p_block->i_buffer,
364                        hh->i_nal_length_size);
365     return p_block;
366 }
367
368 static block_t *
369 helper_process_block_h264_annexb(struct hxxx_helper *hh, block_t *p_block,
370                                  bool *p_config_changed)
371 {
372     if (p_config_changed != NULL)
373     {
374         int i_ret = h264_helper_parse_nal(hh, p_block->p_buffer,
375                                           p_block->i_buffer, 0, p_config_changed);
376         if (i_ret != VLC_SUCCESS)
377         {
378             block_Release(p_block);
379             return NULL;
380         }
381     }
382     return p_block;
383 }
384
385 static block_t *
386 helper_process_block_h264_annexb2avcc(struct hxxx_helper *hh, block_t *p_block,
387                                       bool *p_config_changed)
388 {
389     p_block = helper_process_block_h264_annexb(hh, p_block, p_config_changed);
390     return p_block ? hxxx_AnnexB_to_xVC(p_block, hh->i_nal_length_size) : NULL;
391 }
392
393 static block_t *
394 helper_process_block_dummy(struct hxxx_helper *hh, block_t *p_block,
395                            bool *p_config_changed)
396 {
397     (void) hh;
398     (void) p_config_changed;
399     return p_block;
400 }
401
402 int
403 hxxx_helper_set_extra(struct hxxx_helper *hh, const void *p_extra,
404                       size_t i_extra)
405 {
406     int i_ret;
407     switch (hh->i_codec)
408     {
409         case VLC_CODEC_H264:
410             i_ret = h264_helper_set_extra(hh, p_extra, i_extra);
411             break;
412         case VLC_CODEC_HEVC:
413             i_ret = hevc_helper_set_extra(hh, p_extra, i_extra);
414             break;
415         default:
416             vlc_assert_unreachable();
417     }
418     if (i_ret != VLC_SUCCESS)
419         return i_ret;
420
421     if (hh->b_is_xvcC)
422     {
423         if (hh->b_need_xvcC)
424             hh->pf_process_block = helper_process_block_dummy;
425         else
426             hh->pf_process_block = helper_process_block_xvcc2annexb;
427     }
428     else
429     {
430         switch (hh->i_codec)
431         {
432             case VLC_CODEC_H264:
433                 if (hh->b_need_xvcC)
434                     hh->pf_process_block = helper_process_block_h264_annexb2avcc;
435                 else
436                     hh->pf_process_block = helper_process_block_h264_annexb;
437                 break;
438             case VLC_CODEC_HEVC:
439                 if (hh->b_need_xvcC)
440                     return VLC_EGENERIC; /* TODO */
441                 else
442                     hh->pf_process_block = helper_process_block_dummy;
443                 break;
444             default:
445                 vlc_assert_unreachable();
446         }
447     }
448     return VLC_SUCCESS;;
449 }
450
451
452 block_t *
453 h264_helper_get_annexb_config(struct hxxx_helper *hh)
454 {
455     static const uint8_t annexb_startcode[] = { 0x00, 0x00, 0x00, 0x01 };
456
457     if (hh->h264.i_sps_count == 0 || hh->h264.i_pps_count == 0)
458         return NULL;
459
460     const struct hxxx_helper_nal *pp_nal_lists[] = {
461         hh->h264.sps_list, hh->h264.pps_list };
462     const size_t p_nal_counts[] = { hh->h264.i_sps_count, hh->h264.i_pps_count };
463     const size_t p_nal_maxs[] = { H264_SPS_ID_MAX+1, H264_PPS_ID_MAX+1 };
464
465     block_t *p_block_list = NULL;
466     for (size_t i = 0; i < 2; ++i)
467     {
468         size_t i_nals_size = 0;
469         const struct hxxx_helper_nal *p_nal;
470         HELPER_FOREACH_NAL(p_nal, pp_nal_lists[i], p_nal_counts[i], p_nal_maxs[i])
471         {
472             i_nals_size += p_nal->b->i_buffer + sizeof annexb_startcode;
473         }
474
475         block_t *p_block = block_Alloc(i_nals_size);
476         if (p_block == NULL)
477         {
478             if (p_block_list != NULL)
479                 block_Release(p_block_list);
480             return NULL;
481         }
482
483         p_block->i_buffer = 0;
484         HELPER_FOREACH_NAL(p_nal, pp_nal_lists[i], p_nal_counts[i], p_nal_maxs[i])
485         {
486             memcpy(&p_block->p_buffer[p_block->i_buffer], annexb_startcode,
487                    sizeof annexb_startcode);
488             p_block->i_buffer += sizeof annexb_startcode;
489             memcpy(&p_block->p_buffer[p_block->i_buffer], p_nal->b->p_buffer,
490                    p_nal->b->i_buffer);
491             p_block->i_buffer += p_nal->b->i_buffer;
492         }
493         if (p_block_list == NULL)
494             p_block_list = p_block;
495         else
496             p_block_list->p_next = p_block;
497     }
498
499     return p_block_list;
500 }
501
502 block_t *
503 h264_helper_get_avcc_config(struct hxxx_helper *hh)
504 {
505     const struct hxxx_helper_nal *p_nal;
506     size_t i = 0;
507     const uint8_t *pp_sps_bufs[hh->h264.i_sps_count];
508     size_t p_sps_sizes[hh->h264.i_sps_count];
509     HELPER_FOREACH_NAL(p_nal, hh->h264.sps_list, hh->h264.i_sps_count,
510                        H264_SPS_ID_MAX+1)
511     {
512         pp_sps_bufs[i] = p_nal->b->p_buffer;
513         p_sps_sizes[i] = p_nal->b->i_buffer;
514         ++i;
515     }
516
517     i = 0;
518     const uint8_t *pp_pps_bufs[hh->h264.i_pps_count];
519     size_t p_pps_sizes[hh->h264.i_pps_count];
520     HELPER_FOREACH_NAL(p_nal, hh->h264.pps_list, hh->h264.i_pps_count,
521                        H264_PPS_ID_MAX+1)
522     {
523         pp_pps_bufs[i] = p_nal->b->p_buffer;
524         p_pps_sizes[i] = p_nal->b->i_buffer;
525         ++i;
526     }
527     return h264_NAL_to_avcC(4, pp_sps_bufs, p_sps_sizes, hh->h264.i_sps_count,
528                             pp_pps_bufs, p_pps_sizes, hh->h264.i_pps_count);
529 }
530
531 static const struct hxxx_helper_nal *
532 h264_helper_get_current_sps(struct hxxx_helper *hh)
533 {
534     if (hh->h264.i_sps_count == 0)
535         return NULL;
536
537     const struct hxxx_helper_nal *hsps =
538         &hh->h264.sps_list[hh->h264.i_current_sps];
539     assert(hsps->b != NULL);
540     return hsps;
541 }
542
543 int
544 h264_helper_get_current_picture_size(struct hxxx_helper *hh,
545                                      unsigned *p_w, unsigned *p_h,
546                                      unsigned *p_vw, unsigned *p_vh)
547 {
548     const struct hxxx_helper_nal *hsps = h264_helper_get_current_sps(hh);
549     if (hsps == NULL)
550         return VLC_EGENERIC;
551     return h264_get_picture_size(hsps->h264_sps, p_w, p_h, p_vw, p_vh) ?
552            VLC_SUCCESS : VLC_EGENERIC;
553 }
554
555 int
556 h264_helper_get_current_sar(struct hxxx_helper *hh, int *p_num, int *p_den)
557 {
558     const struct hxxx_helper_nal *hsps = h264_helper_get_current_sps(hh);
559     if (hsps == NULL)
560         return VLC_EGENERIC;
561     *p_num = hsps->h264_sps->vui.i_sar_num;
562     *p_den = hsps->h264_sps->vui.i_sar_den;
563     return VLC_SUCCESS;
564 }
565
566 int
567 h264_helper_get_current_dpb_values(struct hxxx_helper *hh,
568                                        uint8_t *p_depth, unsigned *p_delay)
569 {
570     const struct hxxx_helper_nal *hsps = h264_helper_get_current_sps(hh);
571     if (hsps == NULL)
572         return VLC_EGENERIC;
573     return h264_get_dpb_values(hsps->h264_sps, p_depth, p_delay) ?
574            VLC_SUCCESS : VLC_EGENERIC;
575 }