demux: heif: add preliminary support for AVIF
[vlc.git] / modules / demux / mp4 / heif.c
1 /*****************************************************************************
2  * heif.c : ISO/IEC 23008-12 HEIF still picture demuxer
3  *****************************************************************************
4  * Copyright (C) 2018 Videolabs, VLC authors and VideoLAN
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published by
8  * the Free Software Foundation; either version 2.1 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
19  *****************************************************************************/
20 #ifdef HAVE_CONFIG_H
21 # include "config.h"
22 #endif
23
24 /*****************************************************************************
25  * Preamble
26  *****************************************************************************/
27 #include <vlc_common.h>
28 #include <vlc_demux.h>
29 #include <vlc_input.h>
30 #include <assert.h>
31 #include <limits.h>
32
33 #include "libmp4.h"
34 #include "heif.h"
35 #include "../../packetizer/iso_color_tables.h"
36
37 struct heif_private_t
38 {
39     MP4_Box_t *p_root;
40     es_out_id_t *id;
41     vlc_tick_t i_pcr;
42     vlc_tick_t i_end_display_time;
43     vlc_tick_t i_image_duration;
44     bool b_seekpoint_changed;
45     uint32_t i_seekpoint;
46     input_title_t *p_title;
47
48     struct
49     {
50         MP4_Box_t *p_infe;
51         es_format_t fmt;
52         const MP4_Box_t *p_shared_header;
53     } current;
54 };
55
56 static MP4_Box_t * NextAtom( MP4_Box_t *p_root,
57                              vlc_fourcc_t i_type, const char *psz_path,
58                              MP4_Box_t *p_infe )
59 {
60     if( p_infe == NULL )
61         p_infe = MP4_BoxGet( p_root, psz_path );
62     else
63         p_infe = p_infe->p_next;
64     for( ; p_infe; p_infe = p_infe->p_next )
65     {
66         if( p_infe->i_type == i_type )
67             return p_infe;
68     }
69     return NULL;
70 }
71
72 static MP4_Box_t * GetAtom( MP4_Box_t *p_root, MP4_Box_t *p_atom,
73                             vlc_fourcc_t i_type, const char *psz_path,
74                             bool(*pf_match)(const MP4_Box_t *, void *),
75                             void *priv )
76 {
77     while( (p_atom = NextAtom( p_root, i_type, psz_path, p_atom )) )
78     {
79         if( pf_match( p_atom, priv ) )
80             return p_atom;
81     }
82     return NULL;
83 }
84
85 static bool MatchInfeID( const MP4_Box_t *p_infe, void *priv )
86 {
87     return BOXDATA(p_infe)->i_item_id == *((uint32_t *) priv);
88 }
89
90 static bool MatchPureImage( const MP4_Box_t *p_infe, void *priv )
91 {
92     MP4_Box_t *p_root = priv;
93     const MP4_Box_t *p_iref = MP4_BoxGet( p_root, "meta/iref" );
94     if( !p_iref )
95         return true;
96     for( const MP4_Box_t *p_refbox = p_iref->p_first;
97                           p_refbox; p_refbox = p_refbox->p_next )
98     {
99         if( BOXDATA(p_refbox)->i_from_item_id == BOXDATA(p_infe)->i_item_id )
100             return false;
101     }
102     return true;
103 }
104
105 static void SeekToPrevImageEnd( struct heif_private_t *p_sys, int i_picture )
106 {
107     int i = 0;
108     MP4_Box_t *p_infe = NULL;
109     while( i < i_picture &&
110           (p_infe = NextAtom( p_sys->p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
111     {
112         if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
113                 !MatchPureImage( p_infe, p_sys->p_root ) )
114             continue;
115         i++;
116     }
117     p_sys->current.p_infe = p_infe;
118     p_sys->i_end_display_time = 0;
119     p_sys->i_pcr = i * p_sys->i_image_duration;
120 }
121
122 static int ControlHEIF( demux_t *p_demux, int i_query, va_list args )
123 {
124     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
125
126     switch( i_query )
127     {
128         case DEMUX_CAN_SEEK:
129             *va_arg(args, bool *) = true;
130             return VLC_SUCCESS;
131         case DEMUX_GET_TITLE_INFO:
132         {
133             input_title_t ***ppp_title = va_arg( args, input_title_t *** );
134             int *pi_int = va_arg( args, int* );
135             int *pi_title_offset = va_arg( args, int* );
136             int *pi_seekpoint_offset = va_arg( args, int* );
137
138             if( !p_sys->p_title )
139                 return VLC_EGENERIC;
140
141             *pi_int = 1;
142             *ppp_title = malloc( sizeof( input_title_t*) );
143             (*ppp_title)[0] = vlc_input_title_Duplicate( p_sys->p_title );
144             *pi_title_offset = 0;
145             *pi_seekpoint_offset = 0;
146             return VLC_SUCCESS;
147         }
148         case DEMUX_SET_TITLE:
149         {
150             const int i_title = va_arg( args, int );
151             if( !p_sys->p_title || i_title != 0 )
152                 return VLC_EGENERIC;
153             return VLC_SUCCESS;
154         }
155         case DEMUX_GET_SEEKPOINT:
156             *va_arg( args, int * ) = p_sys->i_seekpoint;
157             return VLC_SUCCESS;
158         case DEMUX_SET_SEEKPOINT:
159         {
160             const int i_seekpoint = va_arg( args, int );
161             if( !p_sys->p_title )
162                 return VLC_EGENERIC;
163             SeekToPrevImageEnd( p_sys, i_seekpoint );
164             return VLC_SUCCESS;
165         }
166         case DEMUX_TEST_AND_CLEAR_FLAGS:
167         {
168             unsigned *restrict flags = va_arg( args, unsigned * );
169
170             if ((*flags & INPUT_UPDATE_SEEKPOINT) && p_sys->b_seekpoint_changed)
171             {
172                 *flags = INPUT_UPDATE_SEEKPOINT;
173                 p_sys->b_seekpoint_changed = false;
174             }
175             else
176                 *flags = 0;
177             return VLC_SUCCESS;
178         }
179         case DEMUX_GET_LENGTH:
180             *(va_arg( args, vlc_tick_t * )) = p_sys->p_title->i_seekpoint *
181                                               p_sys->i_image_duration;
182             return VLC_SUCCESS;
183         case DEMUX_GET_TIME:
184             *(va_arg(args, vlc_tick_t *)) = p_sys->i_pcr;
185             return VLC_SUCCESS;
186         case DEMUX_SET_TIME:
187         {
188             SeekToPrevImageEnd( p_sys, va_arg(args, vlc_tick_t) /
189                                 p_sys->i_image_duration );
190             return VLC_SUCCESS;
191         }
192         case DEMUX_GET_POSITION:
193             if( !p_sys->p_title->i_seekpoint )
194                 return VLC_EGENERIC;
195             *(va_arg(args, double *)) = (double) p_sys->i_pcr /
196                     (p_sys->p_title->i_seekpoint * p_sys->i_image_duration);
197             return VLC_SUCCESS;
198         case DEMUX_SET_POSITION:
199         {
200             SeekToPrevImageEnd( p_sys,  va_arg(args, double) * p_sys->p_title->i_seekpoint );
201             return VLC_SUCCESS;
202         }
203         case DEMUX_CAN_PAUSE:
204         case DEMUX_SET_PAUSE_STATE:
205         case DEMUX_CAN_CONTROL_PACE:
206         case DEMUX_GET_PTS_DELAY:
207             return demux_vaControlHelper( p_demux->s, 0, -1, 0, 1, i_query, args );
208
209         default:
210             return VLC_EGENERIC;
211
212     }
213 }
214
215 static int DemuxHEIF( demux_t *p_demux )
216 {
217     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
218
219     /* Displaying a picture */
220     if( p_sys->i_end_display_time > 0 )
221     {
222         bool b_empty;
223         es_out_Control( p_demux->out, ES_OUT_GET_EMPTY, &b_empty );
224         if( !b_empty || vlc_tick_now() <= p_sys->i_end_display_time )
225         {
226             vlc_tick_sleep( VLC_TICK_FROM_MS(40) );
227             return VLC_DEMUXER_SUCCESS;
228         }
229         p_sys->i_end_display_time = 0;
230     }
231
232     /* Reset prev pic params */
233     p_sys->current.p_shared_header = NULL;
234
235     /* First or next picture */
236     if( !p_sys->current.p_infe )
237     {
238         MP4_Box_t *p_pitm = MP4_BoxGet( p_sys->p_root, "meta/pitm" );
239         if( !p_pitm )
240             return VLC_DEMUXER_EOF;
241
242         p_sys->current.p_infe = GetAtom( p_sys->p_root, NULL,
243                                          ATOM_infe, "meta/iinf/infe",
244                                          MatchInfeID, &BOXDATA(p_pitm)->i_item_id );
245     }
246     else
247     {
248         p_sys->current.p_infe = GetAtom( p_sys->p_root, p_sys->current.p_infe,
249                                          ATOM_infe, "meta/iinf/infe",
250                                          MatchPureImage, p_sys->p_root );
251     }
252
253     if( !p_sys->current.p_infe )
254         return VLC_DEMUXER_EOF;
255
256     const uint32_t i_current_item_id = p_sys->current.BOXDATA(p_infe)->i_item_id;
257     const MP4_Box_t *p_ipco = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipco" );
258     const MP4_Box_t *p_ipma = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipma" );
259     if( !p_ipma || !p_ipco )
260         return VLC_DEMUXER_EOF;
261
262     es_format_t fmt;
263     const char *psz_mime = p_sys->current.BOXDATA(p_infe)->psz_content_type;
264     switch( p_sys->current.BOXDATA(p_infe)->item_type )
265     {
266         case VLC_FOURCC('h','v','c','1'):
267             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_HEVC );
268             break;
269         case VLC_FOURCC('a','v','c','1'):
270             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_H264 );
271             break;
272         case ATOM_av01:
273             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_AV1 );
274             break;
275         case VLC_FOURCC('j','p','e','g'):
276             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_JPEG );
277             break;
278         default:
279             if( psz_mime )
280             {
281                 if( !strcasecmp( "image/jpeg", psz_mime ) )
282                     es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_JPEG );
283                 else if( !strcasecmp( "image/avif", psz_mime ) )
284                     es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_AV1 );
285                 break;
286             }
287             return VLC_DEMUXER_SUCCESS; /* Unsupported picture, goto next */
288     }
289
290     /* Load properties */
291     for( uint32_t i=0; i<BOXDATA(p_ipma)->i_entry_count; i++ )
292     {
293         if( BOXDATA(p_ipma)->p_entries[i].i_item_id != i_current_item_id )
294             continue;
295         for( uint8_t j=0; j<BOXDATA(p_ipma)->p_entries[i].i_association_count; j++ )
296         {
297             if( !BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index )
298                 continue;
299
300             const MP4_Box_t *p_prop = MP4_BoxGet( p_ipco, "./[%u]",
301                 BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index - 1 );
302             if( !p_prop )
303                 continue;
304
305             switch( p_prop->i_type )
306             {
307                 case ATOM_hvcC:
308                 case ATOM_avcC:
309                 case ATOM_av1C:
310                     if( !fmt.p_extra &&
311                        ((fmt.i_codec == VLC_CODEC_HEVC && p_prop->i_type == ATOM_hvcC) ||
312                         (fmt.i_codec == VLC_CODEC_H264 && p_prop->i_type == ATOM_avcC) ||
313                         (fmt.i_codec == VLC_CODEC_AV1  && p_prop->i_type == ATOM_av1C)) )
314                     {
315                         fmt.p_extra = malloc( p_prop->data.p_binary->i_blob );
316                         if( fmt.p_extra )
317                         {
318                             fmt.i_extra = p_prop->data.p_binary->i_blob;
319                             memcpy( fmt.p_extra, p_prop->data.p_binary->p_blob, fmt.i_extra );
320                         }
321                     }
322                     break;
323                 case ATOM_jpeC:
324                     if( fmt.i_codec == VLC_CODEC_JPEG )
325                         p_sys->current.p_shared_header = p_prop;
326                     break;
327                 case ATOM_ispe:
328                     fmt.video.i_visible_width = p_prop->data.p_ispe->i_width;
329                     fmt.video.i_visible_height = p_prop->data.p_ispe->i_height;
330                     break;
331                 case ATOM_pasp:
332                     if( p_prop->data.p_pasp->i_horizontal_spacing &&
333                         p_prop->data.p_pasp->i_vertical_spacing )
334                     {
335                         fmt.video.i_sar_num = p_prop->data.p_pasp->i_horizontal_spacing;
336                         fmt.video.i_sar_den = p_prop->data.p_pasp->i_vertical_spacing;
337                     }
338                     break;
339                 case ATOM_irot:
340                     switch( p_prop->data.p_irot->i_ccw_degrees % 360 )
341                     {
342                         default:
343                         case 0:   fmt.video.orientation = ORIENT_NORMAL ; break;
344                         case 90:  fmt.video.orientation = ORIENT_ROTATED_90; break;
345                         case 180: fmt.video.orientation = ORIENT_ROTATED_180 ; break;
346                         case 270: fmt.video.orientation = ORIENT_ROTATED_270 ; break;
347                     }
348                     break;
349                 case ATOM_colr:
350                     fmt.video.primaries = iso_23001_8_cp_to_vlc_primaries(
351                                             p_prop->data.p_colr->nclc.i_primary_idx );
352                     fmt.video.transfer = iso_23001_8_tc_to_vlc_xfer(
353                                             p_prop->data.p_colr->nclc.i_transfer_function_idx );
354                     fmt.video.space = iso_23001_8_mc_to_vlc_coeffs(
355                                         p_prop->data.p_colr->nclc.i_matrix_idx );
356                     fmt.video.b_color_range_full = p_prop->data.p_colr->nclc.i_full_range;
357                     break;
358                 case ATOM_clli:
359                     fmt.video.lighting.MaxCLL = p_prop->data.p_CoLL->i_maxCLL;
360                     fmt.video.lighting.MaxFALL = p_prop->data.p_CoLL->i_maxFALL;
361                     break;
362                 case ATOM_mdcv:
363                     memcpy( fmt.video.mastering.primaries,
364                             p_prop->data.p_SmDm->primaries, sizeof(uint16_t) * 6 );
365                     memcpy( fmt.video.mastering.white_point,
366                             p_prop->data.p_SmDm->white_point, sizeof(uint16_t) * 2 );
367                     fmt.video.mastering.max_luminance = p_prop->data.p_SmDm->i_luminanceMax;
368                     fmt.video.mastering.min_luminance = p_prop->data.p_SmDm->i_luminanceMin;
369                     break;
370             }
371         }
372     }
373
374     fmt.video.i_frame_rate      = 1000;
375     fmt.video.i_frame_rate_base = p_sys->i_image_duration / 1000;
376
377     es_format_Clean( &p_sys->current.fmt );
378     es_format_Copy( &p_sys->current.fmt, &fmt );
379     es_format_Clean( &fmt );
380     if( !p_sys->id )
381         p_sys->id = es_out_Add( p_demux->out, &p_sys->current.fmt );
382     else
383         es_out_Control( p_demux->out, ES_OUT_RESTART_ES, p_sys->id );
384
385     if( !p_sys->id )
386     {
387         p_sys->current.p_infe = NULL; /* Goto next picture */
388         return VLC_DEMUXER_SUCCESS;
389     }
390
391     MP4_Box_t *p_iloc = MP4_BoxGet( p_sys->p_root, "meta/iloc" );
392     if( !p_iloc )
393         return VLC_DEMUXER_EOF;
394
395     block_t *p_block = NULL;
396     for( uint32_t i=0; i<BOXDATA(p_iloc)->i_item_count; i++ )
397     {
398         if( BOXDATA(p_iloc)->p_items[i].i_item_id != i_current_item_id )
399             continue;
400
401         block_t **pp_append = &p_block;
402
403         /* Shared prefix data, ex: JPEG */
404         if( p_sys->current.p_shared_header )
405         {
406             *pp_append = block_Alloc( p_sys->current.p_shared_header->data.p_binary->i_blob );
407             if( *pp_append )
408             {
409                 memcpy( (*pp_append)->p_buffer,
410                         p_sys->current.p_shared_header->data.p_binary->p_blob,
411                         p_sys->current.p_shared_header->data.p_binary->i_blob );
412                 pp_append = &((*pp_append)->p_next);
413             }
414         }
415
416         for( uint16_t j=0; j<BOXDATA(p_iloc)->p_items[i].i_extent_count; j++ )
417         {
418             uint64_t i_offset = BOXDATA(p_iloc)->p_items[i].i_base_offset +
419                                 BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_offset;
420             uint64_t i_length = BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_length;
421             if( vlc_stream_Seek( p_demux->s, i_offset ) != VLC_SUCCESS )
422                 break;
423             *pp_append = vlc_stream_Block( p_demux->s, i_length );
424             if( *pp_append )
425                 pp_append = &((*pp_append)->p_next);
426         }
427         break;
428     }
429
430     if( p_block )
431         p_block = block_ChainGather( p_block );
432     if( !p_block )
433         return VLC_DEMUXER_SUCCESS; /* Goto next picture */
434
435     if( p_sys->i_pcr == VLC_TICK_INVALID )
436     {
437         p_sys->i_pcr = VLC_TICK_0;
438         es_out_SetPCR( p_demux->out, p_sys->i_pcr );
439     }
440
441     p_block->i_dts = p_block->i_pts = p_sys->i_pcr;
442     p_block->i_length = p_sys->i_image_duration;
443
444     p_block->i_flags |= BLOCK_FLAG_END_OF_SEQUENCE;
445
446     p_sys->i_end_display_time = vlc_tick_now() + p_block->i_length;
447     p_sys->b_seekpoint_changed = true;
448
449     p_sys->i_pcr = p_block->i_dts + p_block->i_length;
450     es_out_Send( p_demux->out, p_sys->id, p_block );
451     es_out_SetPCR( p_demux->out, p_sys->i_pcr );
452
453     return VLC_DEMUXER_SUCCESS;
454 }
455
456 int OpenHEIF( vlc_object_t * p_this )
457 {
458     demux_t  *p_demux = (demux_t *)p_this;
459     const uint8_t *p_peek;
460
461     if( vlc_stream_Peek( p_demux->s, &p_peek, 12 ) < 12 )
462         return VLC_EGENERIC;
463
464     if( VLC_FOURCC( p_peek[4], p_peek[5], p_peek[6], p_peek[7] ) != ATOM_ftyp )
465         return VLC_EGENERIC;
466
467     switch( VLC_FOURCC( p_peek[8], p_peek[9], p_peek[10], p_peek[11] ) )
468     {
469         case BRAND_mif1:
470         case BRAND_heic:
471         case BRAND_heix:
472         case BRAND_jpeg:
473         case BRAND_avci:
474         case BRAND_avif:
475             break;
476         case BRAND_msf1:
477         case BRAND_hevc:
478         case BRAND_hevx:
479         case BRAND_avcs:
480         case BRAND_avis:
481         default:
482             return VLC_EGENERIC;
483     }
484
485     MP4_Box_t *p_root = MP4_BoxGetRoot( p_demux->s );
486     if( !p_root )
487         return VLC_EGENERIC;
488
489     MP4_BoxDumpStructure( p_demux->s, p_root );
490
491     struct heif_private_t *p_sys = calloc( 1, sizeof(*p_sys) );
492     p_demux->p_sys = (void *) p_sys;
493     p_sys->p_root = p_root;
494     p_sys->p_title = vlc_input_title_New();
495     if( !p_sys->p_title )
496     {
497         free( p_sys );
498         return VLC_ENOMEM;
499     }
500
501     p_sys->i_image_duration = vlc_tick_from_sec(var_InheritFloat( p_demux, "heif-image-duration" ));
502     if( p_sys->i_image_duration <= 0 )
503         p_sys->i_image_duration = VLC_TICK_FROM_SEC(HEIF_DEFAULT_DURATION);
504
505     MP4_Box_t *p_infe = NULL;
506     while( (p_infe = NextAtom( p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
507     {
508         if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
509                 !MatchPureImage( p_infe, p_root ) )
510             continue;
511         seekpoint_t *s = vlc_seekpoint_New();
512         if( s )
513         {
514             s->i_time_offset = p_sys->p_title->i_seekpoint * p_sys->i_image_duration;
515             if( BOXDATA(p_infe)->psz_item_name )
516                 s->psz_name = strdup( BOXDATA(p_infe)->psz_item_name );
517             TAB_APPEND( p_sys->p_title->i_seekpoint, p_sys->p_title->seekpoint, s );
518         }
519     }
520
521     es_format_Init( &p_sys->current.fmt, UNKNOWN_ES, 0 );
522
523     p_demux->pf_demux = DemuxHEIF;
524     p_demux->pf_control = ControlHEIF;
525
526     return VLC_SUCCESS;
527 }
528
529 void CloseHEIF ( vlc_object_t * p_this )
530 {
531     demux_t *p_demux = (demux_t *)p_this;
532     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
533     MP4_BoxFree( p_sys->p_root );
534     if( p_sys->id )
535         es_out_Del( p_demux->out, p_sys->id );
536     es_format_Clean( &p_sys->current.fmt );
537     vlc_input_title_Delete( p_sys->p_title );
538     free( p_sys );
539 }