demux: hls: fix byte range parsing
[vlc/vlc-test.git] / modules / demux / hls / playlist / Parser.cpp
1 /*
2  * Parser.cpp
3  *****************************************************************************
4  * Copyright © 2015 - VideoLAN and VLC Authors
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU Lesser General Public License as published
8  * by the Free Software Foundation; either version 2.1 of the License, or
9  * (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with this program; if not, write to the Free Software Foundation,
18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
19  *****************************************************************************/
20 #ifdef HAVE_CONFIG_H
21 # include "config.h"
22 #endif
23
24 #include "Parser.hpp"
25 #include "HLSSegment.hpp"
26 #include "Representation.hpp"
27 #include "../adaptive/playlist/BasePeriod.h"
28 #include "../adaptive/playlist/BaseAdaptationSet.h"
29 #include "../adaptive/playlist/SegmentList.h"
30 #include "../adaptive/tools/Retrieve.hpp"
31 #include "../adaptive/tools/Helper.h"
32 #include "../adaptive/tools/Conversions.hpp"
33 #include "M3U8.hpp"
34 #include "Tags.hpp"
35
36 #include <vlc_strings.h>
37 #include <vlc_stream.h>
38 #include <cstdio>
39 #include <sstream>
40 #include <map>
41 #include <cctype>
42 #include <algorithm>
43
44 using namespace adaptive;
45 using namespace adaptive::playlist;
46 using namespace hls::playlist;
47
48 M3U8Parser::M3U8Parser()
49 {
50 }
51
52 M3U8Parser::~M3U8Parser   ()
53 {
54 }
55
56 static std::list<Tag *> getTagsFromList(std::list<Tag *> &list, int tag)
57 {
58     std::list<Tag *> ret;
59     std::list<Tag *>::const_iterator it;
60     for(it = list.begin(); it != list.end(); ++it)
61     {
62         if( (*it)->getType() == tag )
63             ret.push_back(*it);
64     }
65     return ret;
66 }
67
68 static void releaseTagsList(std::list<Tag *> &list)
69 {
70     std::list<Tag *>::const_iterator it;
71     for(it = list.begin(); it != list.end(); ++it)
72         delete *it;
73     list.clear();
74 }
75
76 void M3U8Parser::setFormatFromCodecs(Representation *rep, const std::string codecsstring)
77 {
78     std::list<std::string> codecs;
79     std::list<std::string> tokens = Helper::tokenize(codecsstring, ',');
80     std::list<std::string>::const_iterator it;
81     for(it=tokens.begin(); it!=tokens.end(); ++it)
82     {
83         /* Truncate init data */
84         std::size_t pos = (*it).find_first_of('.', 0);
85         if(pos != std::string::npos)
86             codecs.push_back((*it).substr(0, pos));
87         else
88             codecs.push_back(*it);
89     }
90
91     if(!codecs.empty())
92     {
93         if(codecs.size() == 1)
94         {
95             std::string codec = codecs.front();
96             transform(codec.begin(), codec.end(), codec.begin(), (int (*)(int))std::tolower);
97             if(codec == "mp4a")
98                 rep->streamFormat = StreamFormat(StreamFormat::PACKEDAAC);
99         }
100         else
101         {
102             rep->streamFormat = StreamFormat(StreamFormat::MPEG2TS);
103         }
104     }
105 }
106
107 void M3U8Parser::setFormatFromExtension(Representation *rep, const std::string &filename)
108 {
109     std::size_t pos = filename.find_last_of('.');
110     if(pos != std::string::npos)
111     {
112         std::string extension = filename.substr(pos + 1);
113         transform(extension.begin(), extension.end(), extension.begin(), (int (*)(int))std::tolower);
114         if(extension == "aac")
115         {
116             rep->streamFormat = StreamFormat(StreamFormat::PACKEDAAC);
117         }
118         else if(extension == "ts" || extension == "mp2t" || extension == "mpeg")
119         {
120             rep->streamFormat = StreamFormat(StreamFormat::MPEG2TS);
121         }
122     }
123 }
124
125 Representation * M3U8Parser::createRepresentation(BaseAdaptationSet *adaptSet, const AttributesTag * tag)
126 {
127     const Attribute *uriAttr = tag->getAttributeByName("URI");
128     const Attribute *bwAttr = tag->getAttributeByName("BANDWIDTH");
129     const Attribute *codecsAttr = tag->getAttributeByName("CODECS");
130     const Attribute *resAttr = tag->getAttributeByName("RESOLUTION");
131
132     Representation *rep = new (std::nothrow) Representation(adaptSet);
133     if(rep)
134     {
135         if(uriAttr)
136         {
137             std::string uri;
138             if(tag->getType() == AttributesTag::EXTXMEDIA)
139             {
140                 uri = uriAttr->quotedString();
141             }
142             else
143             {
144                 uri = uriAttr->value;
145             }
146             rep->setID(uri);
147             rep->setPlaylistUrl(uri);
148             if(uri.find('/') != std::string::npos)
149             {
150                 uri = Helper::getDirectoryPath(uri);
151                 if(!uri.empty())
152                     rep->baseUrl.Set(new Url(uri.append("/")));
153             }
154         }
155
156         if(bwAttr)
157             rep->setBandwidth(bwAttr->decimal());
158
159         if(codecsAttr)
160             setFormatFromCodecs(rep, codecsAttr->quotedString());
161
162         if(resAttr)
163         {
164             std::pair<int, int> res = resAttr->getResolution();
165             if(res.first * res.second)
166             {
167                 rep->setWidth(res.first);
168                 rep->setHeight(res.second);
169             }
170         }
171     }
172
173     return rep;
174 }
175
176 void M3U8Parser::createAndFillRepresentation(vlc_object_t *p_obj, BaseAdaptationSet *adaptSet,
177                                              const AttributesTag *tag,
178                                              const std::list<Tag *> &tagslist)
179 {
180     Representation *rep  = createRepresentation(adaptSet, tag);
181     if(rep)
182     {
183         parseSegments(p_obj, rep, tagslist);
184         adaptSet->addRepresentation(rep);
185     }
186 }
187
188 bool M3U8Parser::appendSegmentsFromPlaylistURI(vlc_object_t *p_obj, Representation *rep)
189 {
190     block_t *p_block = Retrieve::HTTP(p_obj, rep->getPlaylistUrl().toString());
191     if(p_block)
192     {
193         stream_t *substream = stream_MemoryNew(p_obj, p_block->p_buffer, p_block->i_buffer, true);
194         if(substream)
195         {
196             std::list<Tag *> tagslist = parseEntries(substream);
197             stream_Delete(substream);
198
199             parseSegments(p_obj, rep, tagslist);
200
201             releaseTagsList(tagslist);
202             return true;
203         }
204         block_Release(p_block);
205     }
206     return false;
207 }
208
209 void M3U8Parser::parseSegments(vlc_object_t *p_obj, Representation *rep, const std::list<Tag *> &tagslist)
210 {
211     SegmentList *segmentList = new (std::nothrow) SegmentList(rep);
212
213     rep->setTimescale(100);
214     rep->b_loaded = true;
215
216     mtime_t totalduration = 0;
217     mtime_t nzStartTime = 0;
218     mtime_t absReferenceTime = VLC_TS_INVALID;
219     uint64_t sequenceNumber = 0;
220     bool discontinuity = false;
221     std::size_t prevbyterangeoffset = 0;
222     const SingleValueTag *ctx_byterange = NULL;
223     SegmentEncryption encryption;
224     const ValuesListTag *ctx_extinf = NULL;
225
226     std::list<Tag *>::const_iterator it;
227     for(it = tagslist.begin(); it != tagslist.end(); ++it)
228     {
229         const Tag *tag = *it;
230         switch(tag->getType())
231         {
232             /* using static cast as attribute type permits avoiding class check */
233             case SingleValueTag::EXTXMEDIASEQUENCE:
234             {
235                 sequenceNumber = (static_cast<const SingleValueTag*>(tag))->getValue().decimal();
236             }
237             break;
238
239             case ValuesListTag::EXTINF:
240             {
241                 ctx_extinf = static_cast<const ValuesListTag *>(tag);
242             }
243             break;
244
245             case SingleValueTag::URI:
246             {
247                 const SingleValueTag *uritag = static_cast<const SingleValueTag *>(tag);
248                 if(uritag->getValue().value.empty())
249                 {
250                     ctx_extinf = NULL;
251                     ctx_byterange = NULL;
252                     break;
253                 }
254
255                 HLSSegment *segment = new (std::nothrow) HLSSegment(rep, sequenceNumber++);
256                 if(!segment)
257                     break;
258
259                 segment->setSourceUrl(uritag->getValue().value);
260                 if((unsigned)rep->getStreamFormat() == StreamFormat::UNKNOWN)
261                     setFormatFromExtension(rep, uritag->getValue().value);
262
263                 if(ctx_extinf)
264                 {
265                     if(ctx_extinf->getAttributeByName("DURATION"))
266                     {
267                         const mtime_t nzDuration = CLOCK_FREQ * ctx_extinf->getAttributeByName("DURATION")->floatingPoint();
268                         segment->duration.Set(ctx_extinf->getAttributeByName("DURATION")->floatingPoint() * (uint64_t) rep->getTimescale());
269                         segment->startTime.Set(rep->getTimescale().ToScaled(nzStartTime));
270                         nzStartTime += nzDuration;
271                         totalduration += nzDuration;
272
273                         if(absReferenceTime > VLC_TS_INVALID)
274                         {
275                             segment->utcTime = absReferenceTime;
276                             absReferenceTime += nzDuration;
277                         }
278                     }
279                     ctx_extinf = NULL;
280                 }
281
282                 segmentList->addSegment(segment);
283
284                 if(ctx_byterange)
285                 {
286                     std::pair<std::size_t,std::size_t> range = ctx_byterange->getValue().getByteRange();
287                     if(range.first == 0) /* first == size, second = offset */
288                         range.first = prevbyterangeoffset;
289                     prevbyterangeoffset = range.first + range.second;
290                     segment->setByteRange(range.first, prevbyterangeoffset - 1);
291                     ctx_byterange = NULL;
292                 }
293
294                 if(discontinuity)
295                 {
296                     segment->discontinuity = true;
297                     discontinuity = false;
298                 }
299
300                 if(encryption.method != SegmentEncryption::NONE)
301                     segment->setEncryption(encryption);
302             }
303             break;
304
305             case SingleValueTag::EXTXTARGETDURATION:
306                 rep->targetDuration = static_cast<const SingleValueTag *>(tag)->getValue().decimal();
307                 break;
308
309             case SingleValueTag::EXTXPLAYLISTTYPE:
310                 rep->b_live = (static_cast<const SingleValueTag *>(tag)->getValue().value != "VOD");
311                 break;
312
313             case SingleValueTag::EXTXBYTERANGE:
314                 ctx_byterange = static_cast<const SingleValueTag *>(tag);
315                 break;
316
317             case SingleValueTag::EXTXPROGRAMDATETIME:
318                 rep->b_consistent = false;
319                 absReferenceTime = VLC_TS_0 +
320                         UTCTime(static_cast<const SingleValueTag *>(tag)->getValue().value).mtime();
321                 break;
322
323             case AttributesTag::EXTXKEY:
324             {
325                 const AttributesTag *keytag = static_cast<const AttributesTag *>(tag);
326                 if( keytag->getAttributeByName("METHOD") &&
327                     keytag->getAttributeByName("METHOD")->value == "AES-128" &&
328                     keytag->getAttributeByName("URI") )
329                 {
330                     encryption.method = SegmentEncryption::AES_128;
331                     encryption.key.clear();
332
333                     Url keyurl(keytag->getAttributeByName("URI")->quotedString());
334                     if(!keyurl.hasScheme())
335                     {
336                         keyurl.prepend(Helper::getDirectoryPath(rep->getPlaylistUrl().toString()).append("/"));
337                     }
338
339                     block_t *p_block = Retrieve::HTTP(p_obj, keyurl.toString());
340                     if(p_block)
341                     {
342                         if(p_block->i_buffer == 16)
343                         {
344                             encryption.key.resize(16);
345                             memcpy(&encryption.key[0], p_block->p_buffer, 16);
346                         }
347                         block_Release(p_block);
348                     }
349
350                     if(keytag->getAttributeByName("IV"))
351                     {
352                         encryption.iv.clear();
353                         encryption.iv = keytag->getAttributeByName("IV")->hexSequence();
354                     }
355                 }
356                 else
357                 {
358                     /* unsupported or invalid */
359                     encryption.method = SegmentEncryption::NONE;
360                     encryption.key.clear();
361                     encryption.iv.clear();
362                 }
363             }
364             break;
365
366             case Tag::EXTXDISCONTINUITY:
367                 discontinuity  = true;
368                 break;
369
370             case Tag::EXTXENDLIST:
371                 rep->b_live = false;
372                 break;
373         }
374     }
375
376     if(rep->isLive())
377     {
378         rep->getPlaylist()->duration.Set(0);
379     }
380     else if(totalduration > rep->getPlaylist()->duration.Get())
381     {
382         rep->getPlaylist()->duration.Set(totalduration);
383     }
384
385     rep->setSegmentList(segmentList);
386 }
387 M3U8 * M3U8Parser::parse(vlc_object_t *p_object, stream_t *p_stream, const std::string &playlisturl)
388 {
389     char *psz_line = stream_ReadLine(p_stream);
390     if(!psz_line || strcmp(psz_line, "#EXTM3U"))
391     {
392         free(psz_line);
393         return NULL;
394     }
395     free(psz_line);
396
397     M3U8 *playlist = new (std::nothrow) M3U8(p_object);
398     if(!playlist)
399         return NULL;
400
401     if(!playlisturl.empty())
402         playlist->setPlaylistUrl( Helper::getDirectoryPath(playlisturl).append("/") );
403
404     BasePeriod *period = new (std::nothrow) BasePeriod( playlist );
405     if(!period)
406         return playlist;
407
408     std::list<Tag *> tagslist = parseEntries(p_stream);
409     bool b_masterplaylist = !getTagsFromList(tagslist, AttributesTag::EXTXSTREAMINF).empty();
410     if(b_masterplaylist)
411     {
412         std::list<Tag *>::const_iterator it;
413         std::map<std::string, AttributesTag *> groupsmap;
414
415         /* We'll need to create an adaptation set for each media group / alternative rendering
416          * we create a list of playlist being and alternative/group */
417         std::list<Tag *> mediainfotags = getTagsFromList(tagslist, AttributesTag::EXTXMEDIA);
418         for(it = mediainfotags.begin(); it != mediainfotags.end(); ++it)
419         {
420             AttributesTag *tag = dynamic_cast<AttributesTag *>(*it);
421             if(tag && tag->getAttributeByName("URI"))
422             {
423                 std::pair<std::string, AttributesTag *> pair(tag->getAttributeByName("URI")->quotedString(), tag);
424                 groupsmap.insert(pair);
425             }
426         }
427
428         /* Then we parse all playlists uri and add them, except when alternative */
429         BaseAdaptationSet *adaptSet = new (std::nothrow) BaseAdaptationSet(period);
430         if(adaptSet)
431         {
432             std::list<Tag *> streaminfotags = getTagsFromList(tagslist, AttributesTag::EXTXSTREAMINF);
433             for(it = streaminfotags.begin(); it != streaminfotags.end(); ++it)
434             {
435                 AttributesTag *tag = dynamic_cast<AttributesTag *>(*it);
436                 if(tag && tag->getAttributeByName("URI"))
437                 {
438                     if(groupsmap.find(tag->getAttributeByName("URI")->value) == groupsmap.end())
439                     {
440                         /* not a group, belong to default adaptation set */
441                         Representation *rep  = createRepresentation(adaptSet, tag);
442                         if(rep)
443                         {
444                             adaptSet->addRepresentation(rep);
445                         }
446                     }
447                 }
448             }
449             if(!adaptSet->getRepresentations().empty())
450                 period->addAdaptationSet(adaptSet);
451             else
452                 delete adaptSet;
453         }
454
455         /* Finally add all groups */
456         std::map<std::string, AttributesTag *>::const_iterator groupsit;
457         for(groupsit = groupsmap.begin(); groupsit != groupsmap.end(); ++groupsit)
458         {
459             BaseAdaptationSet *altAdaptSet = new (std::nothrow) BaseAdaptationSet(period);
460             if(altAdaptSet)
461             {
462                 std::pair<std::string, AttributesTag *> pair = *groupsit;
463                 Representation *rep  = createRepresentation(altAdaptSet, pair.second);
464                 if(rep)
465                 {
466                     altAdaptSet->addRepresentation(rep);
467                 }
468
469                 if(pair.second->getAttributeByName("NAME"))
470                    altAdaptSet->description.Set(pair.second->getAttributeByName("NAME")->quotedString());
471
472                 /* Subtitles unsupported for now */
473                 if(pair.second->getAttributeByName("TYPE")->value != "AUDIO" &&
474                    pair.second->getAttributeByName("TYPE")->value != "VIDEO")
475                 {
476                     rep->streamFormat = StreamFormat(StreamFormat::UNSUPPORTED);
477                 }
478
479                 if(pair.second->getAttributeByName("LANGUAGE"))
480                 {
481                     std::string lang = pair.second->getAttributeByName("LANGUAGE")->quotedString();
482                     std::size_t pos = lang.find_first_of('-');
483                     if(pos != std::string::npos && pos > 0)
484                         altAdaptSet->addLang(lang.substr(0, pos));
485                     else if (lang.size() < 4)
486                         altAdaptSet->addLang(lang);
487                 }
488
489                 if(!altAdaptSet->getRepresentations().empty())
490                     period->addAdaptationSet(altAdaptSet);
491                 else
492                     delete altAdaptSet;
493             }
494         }
495
496     }
497     else /* Non master playlist (opened directly subplaylist or HLS v1) */
498     {
499         BaseAdaptationSet *adaptSet = new (std::nothrow) BaseAdaptationSet(period);
500         if(adaptSet)
501         {
502             period->addAdaptationSet(adaptSet);
503             AttributesTag *tag = new AttributesTag(AttributesTag::EXTXSTREAMINF, "");
504             tag->addAttribute(new Attribute("URI", playlisturl));
505             createAndFillRepresentation(p_object, adaptSet, tag, tagslist);
506             delete tag;
507         }
508     }
509
510     playlist->addPeriod(period);
511
512     releaseTagsList(tagslist);
513
514     playlist->debug();
515     return playlist;
516 }
517
518 std::list<Tag *> M3U8Parser::parseEntries(stream_t *stream)
519 {
520     std::list<Tag *> entrieslist;
521     Tag *lastTag = NULL;
522     char *psz_line;
523
524     while((psz_line = stream_ReadLine(stream)))
525     {
526         if(*psz_line == '#')
527         {
528             if(!strncmp(psz_line, "#EXT", 4)) //tag
529             {
530                 std::string key;
531                 std::string attributes;
532                 const char *split = strchr(psz_line, ':');
533                 if(split)
534                 {
535                     key = std::string(psz_line + 1, split - psz_line - 1);
536                     attributes = std::string(split + 1);
537                 }
538                 else
539                 {
540                     key = std::string(psz_line + 1);
541                 }
542
543                 if(!key.empty())
544                 {
545                     Tag *tag = TagFactory::createTagByName(key, attributes);
546                     if(tag)
547                         entrieslist.push_back(tag);
548                     lastTag = tag;
549                 }
550             }
551         }
552         else if(*psz_line)
553         {
554             /* URI */
555             if(lastTag && lastTag->getType() == AttributesTag::EXTXSTREAMINF)
556             {
557                 AttributesTag *streaminftag = static_cast<AttributesTag *>(lastTag);
558                 /* master playlist uri, merge as attribute */
559                 Attribute *uriAttr = new (std::nothrow) Attribute("URI", std::string(psz_line));
560                 if(uriAttr)
561                     streaminftag->addAttribute(uriAttr);
562             }
563             else /* playlist tag, will take modifiers */
564             {
565                 Tag *tag = TagFactory::createTagByName("", std::string(psz_line));
566                 if(tag)
567                     entrieslist.push_back(tag);
568             }
569             lastTag = NULL;
570         }
571         else // drop
572         {
573             lastTag = NULL;
574         }
575
576         free(psz_line);
577     }
578
579     return entrieslist;
580 }