63922509fe4ac37fbcbf1e57d9d0c22933e5d6bf
[vlc.git] / modules / codec / omxil / mediacodec.c
1 /*****************************************************************************
2  * mediacodec.c: Video decoder module using the Android MediaCodec API
3  *****************************************************************************
4  * Copyright (C) 2012 Martin Storsjo
5  *
6  * Authors: Martin Storsjo <martin@martin.st>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU Lesser General Public License as published by
10  * the Free Software Foundation; either version 2.1 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21  *****************************************************************************/
22
23 /*****************************************************************************
24  * Preamble
25  *****************************************************************************/
26 #ifdef HAVE_CONFIG_H
27 # include "config.h"
28 #endif
29
30 #include <jni.h>
31 #include <stdint.h>
32 #include <assert.h>
33
34 #include <vlc_common.h>
35 #include <vlc_aout.h>
36 #include <vlc_plugin.h>
37 #include <vlc_codec.h>
38 #include <vlc_block_helper.h>
39 #include <vlc_cpu.h>
40 #include <vlc_memory.h>
41 #include <vlc_timestamp_helper.h>
42 #include <vlc_threads.h>
43
44 #include "mediacodec.h"
45 #include "../../packetizer/h264_nal.h"
46 #include "../../packetizer/hevc_nal.h"
47 #include "../../packetizer/hxxx_nal.h"
48 #include <OMX_Core.h>
49 #include <OMX_Component.h>
50 #include "omxil_utils.h"
51 #include "../../video_output/android/android_window.h"
52
53 #define BLOCK_FLAG_CSD (0x01 << BLOCK_FLAG_PRIVATE_SHIFT)
54
55 /* Codec Specific Data */
56 struct csd
57 {
58     uint8_t *p_buf;
59     size_t i_size;
60 };
61
62 #define NEWBLOCK_FLAG_RESTART (0x01)
63 #define NEWBLOCK_FLAG_FLUSH (0x02)
64 /**
65  * Callback called when a new block is processed from DecodeCommon.
66  * It returns -1 in case of error, 0 if block should be dropped, 1 otherwise.
67  */
68 typedef int (*dec_on_new_block_cb)(decoder_t *, block_t *, int *);
69
70 /**
71  * Callback called when decoder is flushing.
72  */
73 typedef void (*dec_on_flush_cb)(decoder_t *);
74
75 /**
76  * Callback called when DecodeCommon try to get an output buffer (pic or block).
77  * It returns -1 in case of error, or the number of output buffer returned.
78  */
79 typedef int (*dec_process_output_cb)(decoder_t *, mc_api_out *, picture_t **, block_t **);
80
81 struct decoder_sys_t
82 {
83     mc_api *api;
84
85     /* Codec Specific Data buffer: sent in DecodeCommon after a start or a flush
86      * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
87     block_t **pp_csd;
88     size_t i_csd_count;
89     size_t i_csd_send;
90
91     bool b_update_format;
92     bool b_has_format;
93
94     int64_t i_preroll_end;
95     int     i_quirks;
96
97     /* Specific Audio/Video callbacks */
98     dec_on_new_block_cb     pf_on_new_block;
99     dec_on_flush_cb         pf_on_flush;
100     dec_process_output_cb   pf_process_output;
101
102     vlc_mutex_t     lock;
103     vlc_thread_t    out_thread;
104     /* Cond used to signal the output thread */
105     vlc_cond_t      cond;
106     /* Cond used to signal the decoder thread */
107     vlc_cond_t      dec_cond;
108     /* Set to true by pf_flush to signal the output thread to flush */
109     bool            b_flush_out;
110     /* If true, the output thread will start to dequeue output pictures */
111     bool            b_output_ready;
112     /* If true, the first input block was successfully dequeued */
113     bool            b_input_dequeued;
114     bool            b_aborted;
115     /* TODO: remove */
116     bool            b_error_signaled;
117
118     union
119     {
120         struct
121         {
122             AWindowHandler *p_awh;
123             unsigned int i_stride, i_slice_height, i_width, i_height;
124             int i_pixel_format;
125             uint8_t i_nal_length_size;
126             size_t i_h264_profile;
127             ArchitectureSpecificCopyData ascd;
128             /* stores the inflight picture for each output buffer or NULL */
129             picture_sys_t** pp_inflight_pictures;
130             unsigned int i_inflight_pictures;
131             timestamp_fifo_t *timestamp_fifo;
132         } video;
133         struct {
134             date_t i_end_date;
135             int i_channels;
136             bool b_extract;
137             /* Some audio decoders need a valid channel count */
138             bool b_need_channels;
139             int pi_extraction[AOUT_CHAN_MAX];
140         } audio;
141     } u;
142 };
143
144 /*****************************************************************************
145  * Local prototypes
146  *****************************************************************************/
147 static int  OpenDecoderJni(vlc_object_t *);
148 static int  OpenDecoderNdk(vlc_object_t *);
149 static void CleanDecoder(decoder_t *);
150 static void CloseDecoder(vlc_object_t *);
151
152 static int Video_OnNewBlock(decoder_t *, block_t *, int *);
153 static void Video_OnFlush(decoder_t *);
154 static int Video_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
155 static picture_t *DecodeVideo(decoder_t *, block_t **);
156
157 static int Audio_OnNewBlock(decoder_t *, block_t *, int *);
158 static void Audio_OnFlush(decoder_t *);
159 static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
160 static block_t *DecodeAudio(decoder_t *, block_t **);
161
162 static void DecodeFlushLocked(decoder_t *);
163 static void DecodeFlush(decoder_t *);
164 static void StopMediaCodec(decoder_t *);
165 static void *OutThread(void *);
166
167 static void InvalidateAllPictures(decoder_t *);
168 static void RemoveInflightPictures(decoder_t *);
169
170 /*****************************************************************************
171  * Module descriptor
172  *****************************************************************************/
173 #define DIRECTRENDERING_TEXT N_("Android direct rendering")
174 #define DIRECTRENDERING_LONGTEXT N_(\
175         "Enable Android direct rendering using opaque buffers.")
176
177 #define MEDIACODEC_AUDIO_TEXT "Use MediaCodec for audio decoding"
178 #define MEDIACODEC_AUDIO_LONGTEXT "Still experimental."
179
180 #define MEDIACODEC_TUNNELEDPLAYBACK_TEXT "Use a tunneled surface for playback"
181
182 #define CFG_PREFIX "mediacodec-"
183
184 vlc_module_begin ()
185     set_description( N_("Video decoder using Android MediaCodec via NDK") )
186     set_category( CAT_INPUT )
187     set_subcategory( SUBCAT_INPUT_VCODEC )
188     set_section( N_("Decoding") , NULL )
189     set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
190     add_bool(CFG_PREFIX "dr", true,
191              DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
192     add_bool(CFG_PREFIX "audio", false,
193              MEDIACODEC_AUDIO_TEXT, MEDIACODEC_AUDIO_LONGTEXT, true)
194     add_bool(CFG_PREFIX "tunneled-playback", false,
195              MEDIACODEC_TUNNELEDPLAYBACK_TEXT, NULL, true)
196     set_callbacks( OpenDecoderNdk, CloseDecoder )
197     add_shortcut( "mediacodec_ndk" )
198     add_submodule ()
199         set_description( N_("Video decoder using Android MediaCodec via JNI") )
200         set_capability( "decoder", 0 )
201         set_callbacks( OpenDecoderJni, CloseDecoder )
202         add_shortcut( "mediacodec_jni" )
203 vlc_module_end ()
204
205 static void CSDFree(decoder_t *p_dec)
206 {
207     decoder_sys_t *p_sys = p_dec->p_sys;
208
209     if (p_sys->pp_csd)
210     {
211         for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
212             block_Release(p_sys->pp_csd[i]);
213         free(p_sys->pp_csd);
214         p_sys->pp_csd = NULL;
215     }
216     p_sys->i_csd_count = 0;
217 }
218
219 /* Create the p_sys->p_csd that will be sent from DecodeCommon */
220 static int CSDDup(decoder_t *p_dec, const struct csd *p_csd, size_t i_count)
221 {
222     decoder_sys_t *p_sys = p_dec->p_sys;
223
224     CSDFree(p_dec);
225
226     p_sys->pp_csd = malloc(i_count * sizeof(block_t *));
227     if (!p_sys->pp_csd)
228         return VLC_ENOMEM;
229
230     for (size_t i = 0; i < i_count; ++i)
231     {
232         p_sys->pp_csd[i] = block_Alloc(p_csd[i].i_size);
233         if (!p_sys->pp_csd[i])
234         {
235             CSDFree(p_dec);
236             return VLC_ENOMEM;
237         }
238         p_sys->pp_csd[i]->i_flags = BLOCK_FLAG_CSD;
239         memcpy(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf, p_csd[i].i_size);
240         p_sys->i_csd_count++;
241     }
242
243     p_sys->i_csd_send = 0;
244     return VLC_SUCCESS;
245 }
246
247 static bool CSDCmp(decoder_t *p_dec, struct csd *p_csd, size_t i_csd_count)
248 {
249     decoder_sys_t *p_sys = p_dec->p_sys;
250
251     if (p_sys->i_csd_count != i_csd_count)
252         return false;
253     for (size_t i = 0; i < i_csd_count; ++i)
254     {
255         if (p_sys->pp_csd[i]->i_buffer != p_csd[i].i_size
256          || memcmp(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf,
257                    p_csd[i].i_size) != 0)
258             return false;
259     }
260     return true;
261 }
262
263 /* Fill the p_sys->p_csd struct with H264 Parameter Sets */
264 static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
265                       bool *p_size_changed)
266 {
267     decoder_sys_t *p_sys = p_dec->p_sys;
268     uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
269     size_t i_sps_size = 0, i_pps_size = 0;
270
271     /* Check if p_buf contains a valid SPS PPS */
272     if (h264_get_spspps(p_buf, i_size, &p_sps_buf, &i_sps_size,
273                         &p_pps_buf, &i_pps_size) == 0 )
274     {
275         struct csd csd[2];
276         int i_csd_count = 0;
277
278         const uint8_t *p_buffer = p_sps_buf;
279         size_t i_buffer = i_sps_size;
280         if(!hxxx_strip_AnnexB_startcode(&p_buffer, &i_buffer))
281             return VLC_EGENERIC;
282
283         h264_sequence_parameter_set_t *p_sps = h264_decode_sps(p_buffer, i_buffer, true);
284         if( !p_sps )
285             return VLC_EGENERIC;
286
287         unsigned vsize[4];
288         (void) h264_get_picture_size( p_sps, &vsize[0], &vsize[1], &vsize[2], &vsize[3] );
289         /* FIXME: what to do with visible width/height ? */
290
291         if (i_sps_size)
292         {
293             csd[i_csd_count].p_buf = p_sps_buf;
294             csd[i_csd_count].i_size = i_sps_size;
295             i_csd_count++;
296         }
297         if (i_pps_size)
298         {
299             csd[i_csd_count].p_buf = p_pps_buf;
300             csd[i_csd_count].i_size = i_pps_size;
301             i_csd_count++;
302         }
303
304         /* Compare the SPS PPS with the old one */
305         if (!CSDCmp(p_dec, csd, i_csd_count))
306         {
307             msg_Warn(p_dec, "New SPS/PPS found, id: %d size: %ux%u sps: %d pps: %d",
308                      p_sps->i_id, vsize[0], vsize[1],
309                      i_sps_size, i_pps_size);
310
311             /* In most use cases, p_sys->p_csd[0] contains a SPS, and
312              * p_sys->p_csd[1] contains a PPS */
313             if (CSDDup(p_dec, csd, i_csd_count))
314             {
315                 h264_release_sps( p_sps );
316                 return VLC_ENOMEM;
317             }
318
319             if (p_size_changed)
320                 *p_size_changed = (vsize[0] != p_sys->u.video.i_width
321                                 || vsize[1] != p_sys->u.video.i_height);
322
323             p_sys->u.video.i_width = vsize[0];
324             p_sys->u.video.i_height = vsize[1];
325
326             h264_release_sps( p_sps );
327
328             return VLC_SUCCESS;
329         }
330
331         h264_release_sps( p_sps );
332     }
333
334     return VLC_EGENERIC;
335 }
336
337 static int ParseVideoExtra(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
338 {
339     decoder_sys_t *p_sys = p_dec->p_sys;
340
341     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
342      || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
343     {
344         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
345         {
346             if (h264_isavcC(p_extra, i_extra))
347             {
348                 size_t i_size = 0;
349                 uint8_t *p_buf = h264_avcC_to_AnnexB_NAL(p_extra, i_extra, &i_size,
350                                                          &p_sys->u.video.i_nal_length_size);
351                 if(p_buf)
352                 {
353                     H264SetCSD(p_dec, p_buf, i_size, NULL);
354                     free(p_buf);
355                 }
356             }
357             else
358                 H264SetCSD(p_dec, p_extra, i_extra, NULL);
359         }
360         else  /* FIXME and refactor: CSDDup vs CSDless SetCSD */
361         {
362             if (hevc_ishvcC(p_extra, i_extra))
363             {
364                 struct csd csd;
365                 csd.p_buf = hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra, &csd.i_size,
366                                                     &p_sys->u.video.i_nal_length_size);
367                 if(csd.p_buf)
368                 {
369                     CSDDup(p_dec, &csd, 1);
370                     free(csd.p_buf);
371                 }
372             }
373             /* FIXME: what to do with AnnexB ? */
374         }
375     }
376     return VLC_SUCCESS;
377 }
378
379 /*****************************************************************************
380  * StartMediaCodec: Create the mediacodec instance
381  *****************************************************************************/
382 static int StartMediaCodec(decoder_t *p_dec)
383 {
384     decoder_sys_t *p_sys = p_dec->p_sys;
385     int i_ret = 0;
386     union mc_api_args args;
387
388     if (p_dec->fmt_in.i_extra && !p_sys->pp_csd)
389     {
390         /* Try first to configure specific Video CSD */
391         if (p_dec->fmt_in.i_cat == VIDEO_ES)
392             i_ret = ParseVideoExtra(p_dec, p_dec->fmt_in.p_extra,
393                                     p_dec->fmt_in.i_extra);
394
395         if (i_ret != VLC_SUCCESS)
396             return i_ret;
397
398         /* Set default CSD if ParseVideoExtra failed to configure one */
399         if (!p_sys->pp_csd)
400         {
401             struct csd csd;
402
403             csd.p_buf = p_dec->fmt_in.p_extra;
404             csd.i_size = p_dec->fmt_in.i_extra;
405             CSDDup(p_dec, &csd, 1);
406         }
407     }
408
409     if (p_dec->fmt_in.i_cat == VIDEO_ES)
410     {
411         if (!p_sys->u.video.i_width || !p_sys->u.video.i_height)
412         {
413             msg_Err(p_dec, "invalid size, abort MediaCodec");
414             return VLC_EGENERIC;
415         }
416         args.video.i_width = p_sys->u.video.i_width;
417         args.video.i_height = p_sys->u.video.i_height;
418
419         switch (p_dec->fmt_in.video.orientation)
420         {
421             case ORIENT_ROTATED_90:
422                 args.video.i_angle = 90;
423                 break;
424             case ORIENT_ROTATED_180:
425                 args.video.i_angle = 180;
426                 break;
427             case ORIENT_ROTATED_270:
428                 args.video.i_angle = 270;
429                 break;
430             default:
431                 args.video.i_angle = 0;
432         }
433
434         /* Check again the codec name if h264 profile changed */
435         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
436          && !p_sys->u.video.i_h264_profile)
437         {
438             h264_get_profile_level(&p_dec->fmt_in,
439                                    &p_sys->u.video.i_h264_profile, NULL, NULL);
440             if (p_sys->u.video.i_h264_profile)
441             {
442                 free(p_sys->api->psz_name);
443                 p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
444                                                           p_sys->api->psz_mime,
445                                                           p_sys->u.video.i_h264_profile);
446                 if (!p_sys->api->psz_name)
447                     return VLC_EGENERIC;
448             }
449         }
450
451         if (!p_sys->u.video.p_awh && var_InheritBool(p_dec, CFG_PREFIX "dr"))
452         {
453             if ((p_sys->u.video.p_awh = AWindowHandler_new(VLC_OBJECT(p_dec))))
454             {
455                 /* Direct rendering:
456                  * The surface must be released by the Vout before calling
457                  * start. Request a valid OPAQUE Vout to release any non-OPAQUE
458                  * Vout that will release the surface.
459                  */
460                 p_dec->fmt_out.video.i_width = p_sys->u.video.i_width;
461                 p_dec->fmt_out.video.i_height = p_sys->u.video.i_height;
462                 p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
463                 if (decoder_UpdateVideoFormat(p_dec) != 0)
464                 {
465                     msg_Err(p_dec, "Opaque Vout request failed: "
466                                    "fallback to non opaque");
467
468                     AWindowHandler_destroy(p_sys->u.video.p_awh);
469                     p_sys->u.video.p_awh = NULL;
470                 }
471             }
472         }
473         args.video.p_awh = p_sys->u.video.p_awh;
474         args.video.b_tunneled_playback = args.video.p_awh ?
475                 var_InheritBool(p_dec, CFG_PREFIX "tunneled-playback") : false;
476     }
477     else
478     {
479         date_Set(&p_sys->u.audio.i_end_date, VLC_TS_INVALID);
480
481         args.audio.i_sample_rate    = p_dec->fmt_in.audio.i_rate;
482         args.audio.i_channel_count  = p_dec->p_sys->u.audio.i_channels;
483     }
484
485     return p_sys->api->start(p_sys->api, &args);
486 }
487
488 /*****************************************************************************
489  * StopMediaCodec: Close the mediacodec instance
490  *****************************************************************************/
491 static void StopMediaCodec(decoder_t *p_dec)
492 {
493     decoder_sys_t *p_sys = p_dec->p_sys;
494
495     /* Remove all pictures that are currently in flight in order
496      * to prevent the vout from using destroyed output buffers. */
497     if (p_sys->api->b_direct_rendering)
498         RemoveInflightPictures(p_dec);
499
500     p_sys->api->stop(p_sys->api);
501     if (p_dec->fmt_in.i_cat == VIDEO_ES && p_sys->u.video.p_awh)
502         AWindowHandler_releaseSurface(p_sys->u.video.p_awh, AWindow_Video);
503 }
504
505 /*****************************************************************************
506  * OpenDecoder: Create the decoder instance
507  *****************************************************************************/
508 static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
509 {
510     decoder_t *p_dec = (decoder_t *)p_this;
511     decoder_sys_t *p_sys;
512     mc_api *api;
513     const char *mime = NULL;
514     bool b_late_opening = false;
515
516     /* Video or Audio if "mediacodec-audio" bool is true */
517     if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
518      || !var_InheritBool(p_dec, CFG_PREFIX "audio")))
519         return VLC_EGENERIC;
520
521     if (p_dec->fmt_in.i_cat == VIDEO_ES)
522     {
523         if (!p_dec->fmt_in.video.i_width || !p_dec->fmt_in.video.i_height)
524         {
525             /* We can handle h264 without a valid video size */
526             if (p_dec->fmt_in.i_codec != VLC_CODEC_H264)
527             {
528                 msg_Dbg(p_dec, "resolution (%dx%d) not supported",
529                         p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
530                 return VLC_EGENERIC;
531             }
532         }
533
534         switch (p_dec->fmt_in.i_codec) {
535         case VLC_CODEC_HEVC: mime = "video/hevc"; break;
536         case VLC_CODEC_H264: mime = "video/avc"; break;
537         case VLC_CODEC_H263: mime = "video/3gpp"; break;
538         case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
539         case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
540         case VLC_CODEC_VC1:  mime = "video/wvc1"; break;
541         case VLC_CODEC_VP8:  mime = "video/x-vnd.on2.vp8"; break;
542         case VLC_CODEC_VP9:  mime = "video/x-vnd.on2.vp9"; break;
543         /* case VLC_CODEC_MPGV: mime = "video/mpeg2"; break; */
544         }
545     }
546     else
547     {
548         switch (p_dec->fmt_in.i_codec) {
549         case VLC_CODEC_AMR_NB: mime = "audio/3gpp"; break;
550         case VLC_CODEC_AMR_WB: mime = "audio/amr-wb"; break;
551         case VLC_CODEC_MPGA:
552         case VLC_CODEC_MP3:    mime = "audio/mpeg"; break;
553         case VLC_CODEC_MP2:    mime = "audio/mpeg-L2"; break;
554         case VLC_CODEC_MP4A:   mime = "audio/mp4a-latm"; break;
555         case VLC_CODEC_QCELP:  mime = "audio/qcelp"; break;
556         case VLC_CODEC_VORBIS: mime = "audio/vorbis"; break;
557         case VLC_CODEC_OPUS:   mime = "audio/opus"; break;
558         case VLC_CODEC_ALAW:   mime = "audio/g711-alaw"; break;
559         case VLC_CODEC_MULAW:  mime = "audio/g711-mlaw"; break;
560         case VLC_CODEC_FLAC:   mime = "audio/flac"; break;
561         case VLC_CODEC_GSM:    mime = "audio/gsm"; break;
562         case VLC_CODEC_A52:    mime = "audio/ac3"; break;
563         case VLC_CODEC_EAC3:   mime = "audio/eac3"; break;
564         case VLC_CODEC_ALAC:   mime = "audio/alac"; break;
565         case VLC_CODEC_DTS:    mime = "audio/vnd.dts"; break;
566         /* case VLC_CODEC_: mime = "audio/mpeg-L1"; break; */
567         /* case VLC_CODEC_: mime = "audio/aac-adts"; break; */
568         }
569     }
570     if (!mime)
571     {
572         msg_Dbg(p_dec, "codec %4.4s not supported",
573                 (char *)&p_dec->fmt_in.i_codec);
574         return VLC_EGENERIC;
575     }
576
577     api = calloc(1, sizeof(mc_api));
578     if (!api)
579         return VLC_ENOMEM;
580     api->p_obj = p_this;
581     api->b_video = p_dec->fmt_in.i_cat == VIDEO_ES;
582     if (pf_init(api) != VLC_SUCCESS)
583     {
584         free(api);
585         return VLC_EGENERIC;
586     }
587
588     /* Allocate the memory needed to store the decoder's structure */
589     if ((p_sys = calloc(1, sizeof(*p_sys))) == NULL)
590     {
591         api->clean(api);
592         free(api);
593         return VLC_ENOMEM;
594     }
595     p_sys->api = api;
596     p_dec->p_sys = p_sys;
597
598     p_dec->pf_decode_video = DecodeVideo;
599     p_dec->pf_decode_audio = DecodeAudio;
600     p_dec->pf_flush        = DecodeFlush;
601
602     p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
603     p_dec->fmt_out.video = p_dec->fmt_in.video;
604     p_dec->fmt_out.audio = p_dec->fmt_in.audio;
605     p_sys->api->psz_mime = mime;
606
607     vlc_mutex_init(&p_sys->lock);
608     vlc_cond_init(&p_sys->cond);
609     vlc_cond_init(&p_sys->dec_cond);
610
611     if (p_dec->fmt_in.i_cat == VIDEO_ES)
612     {
613         p_sys->pf_on_new_block = Video_OnNewBlock;
614         p_sys->pf_on_flush = Video_OnFlush;
615         p_sys->pf_process_output = Video_ProcessOutput;
616         p_sys->u.video.i_width = p_dec->fmt_in.video.i_width;
617         p_sys->u.video.i_height = p_dec->fmt_in.video.i_height;
618
619         p_sys->u.video.timestamp_fifo = timestamp_FifoNew(32);
620         if (!p_sys->u.video.timestamp_fifo)
621             goto bailout;
622         TAB_INIT( p_sys->u.video.i_inflight_pictures,
623                   p_sys->u.video.pp_inflight_pictures );
624
625         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
626             h264_get_profile_level(&p_dec->fmt_in,
627                                    &p_sys->u.video.i_h264_profile, NULL, NULL);
628
629         p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
630                                                   p_sys->api->psz_mime,
631                                                   p_sys->u.video.i_h264_profile);
632         if (!p_sys->api->psz_name)
633             goto bailout;
634
635         p_sys->i_quirks = OMXCodec_GetQuirks(VIDEO_ES,
636                                              p_dec->fmt_in.i_codec,
637                                              p_sys->api->psz_name,
638                                              strlen(p_sys->api->psz_name));
639
640         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_NEED_SIZE)
641          && (!p_sys->u.video.i_width || !p_sys->u.video.i_height))
642         {
643             msg_Warn(p_dec, "waiting for a valid video size for codec %4.4s",
644                      (const char *)&p_dec->fmt_in.i_codec);
645             b_late_opening = true;
646         }
647     }
648     else
649     {
650         p_sys->pf_on_new_block = Audio_OnNewBlock;
651         p_sys->pf_on_flush = Audio_OnFlush;
652         p_sys->pf_process_output = Audio_ProcessOutput;
653         p_sys->u.audio.i_channels = p_dec->fmt_in.audio.i_channels;
654
655         p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
656                                                   p_sys->api->psz_mime, 0);
657         if (!p_sys->api->psz_name)
658             goto bailout;
659
660         p_sys->i_quirks = OMXCodec_GetQuirks(AUDIO_ES,
661                                              p_dec->fmt_in.i_codec,
662                                              p_sys->api->psz_name,
663                                              strlen(p_sys->api->psz_name));
664         if ((p_sys->i_quirks & OMXCODEC_AUDIO_QUIRKS_NEED_CHANNELS)
665          && !p_sys->u.audio.i_channels)
666         {
667             msg_Warn(p_dec, "waiting for valid channel count");
668             b_late_opening = true;
669         }
670     }
671     if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
672      && !p_dec->fmt_in.i_extra)
673     {
674         msg_Warn(p_dec, "waiting for extra data for codec %4.4s",
675                  (const char *)&p_dec->fmt_in.i_codec);
676         if (p_dec->fmt_in.i_codec == VLC_CODEC_MP4V)
677         {
678             msg_Warn(p_dec, "late opening with MPEG4 not handled"); /* TODO */
679             goto bailout;
680         }
681         b_late_opening = true;
682     }
683
684     if (!b_late_opening && StartMediaCodec(p_dec) != VLC_SUCCESS)
685     {
686         msg_Err(p_dec, "StartMediaCodec failed");
687         goto bailout;
688     }
689
690     if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
691                   VLC_THREAD_PRIORITY_LOW))
692     {
693         msg_Err(p_dec, "vlc_clone failed");
694         vlc_mutex_unlock(&p_sys->lock);
695         goto bailout;
696     }
697
698     return VLC_SUCCESS;
699
700 bailout:
701     CleanDecoder(p_dec);
702     return VLC_EGENERIC;
703 }
704
705 static int OpenDecoderNdk(vlc_object_t *p_this)
706 {
707     return OpenDecoder(p_this, MediaCodecNdk_Init);
708 }
709
710 static int OpenDecoderJni(vlc_object_t *p_this)
711 {
712     return OpenDecoder(p_this, MediaCodecJni_Init);
713 }
714
715 static void AbortDecoderLocked(decoder_t *p_dec)
716 {
717     decoder_sys_t *p_sys = p_dec->p_sys;
718
719     if (!p_sys->b_aborted)
720     {
721         p_sys->b_aborted = true;
722         vlc_cancel(p_sys->out_thread);
723     }
724 }
725
726 static void CleanDecoder(decoder_t *p_dec)
727 {
728     decoder_sys_t *p_sys = p_dec->p_sys;
729
730     vlc_mutex_destroy(&p_sys->lock);
731     vlc_cond_destroy(&p_sys->cond);
732     vlc_cond_destroy(&p_sys->dec_cond);
733
734     StopMediaCodec(p_dec);
735
736     CSDFree(p_dec);
737     p_sys->api->clean(p_sys->api);
738
739     if (p_dec->fmt_in.i_cat == VIDEO_ES)
740     {
741         ArchitectureSpecificCopyHooksDestroy(p_sys->u.video.i_pixel_format,
742                                              &p_sys->u.video.ascd);
743         if (p_sys->u.video.timestamp_fifo)
744             timestamp_FifoRelease(p_sys->u.video.timestamp_fifo);
745         if (p_sys->u.video.p_awh)
746             AWindowHandler_destroy(p_sys->u.video.p_awh);
747     }
748     free(p_sys->api->psz_name);
749     free(p_sys->api);
750     free(p_sys);
751 }
752
753 /*****************************************************************************
754  * CloseDecoder: Close the decoder instance
755  *****************************************************************************/
756 static void CloseDecoder(vlc_object_t *p_this)
757 {
758     decoder_t *p_dec = (decoder_t *)p_this;
759     decoder_sys_t *p_sys = p_dec->p_sys;
760
761     vlc_mutex_lock(&p_sys->lock);
762     /* Unblock output thread waiting in dequeue_out */
763     DecodeFlushLocked(p_dec);
764     /* Cancel the output thread */
765     AbortDecoderLocked(p_dec);
766     vlc_mutex_unlock(&p_sys->lock);
767
768     vlc_join(p_sys->out_thread, NULL);
769
770     CleanDecoder(p_dec);
771 }
772
773 /*****************************************************************************
774  * vout callbacks
775  *****************************************************************************/
776 static void ReleasePicture(decoder_t *p_dec, unsigned i_index, bool b_render)
777 {
778     decoder_sys_t *p_sys = p_dec->p_sys;
779
780     p_sys->api->release_out(p_sys->api, i_index, b_render);
781 }
782
783 static void InvalidateAllPictures(decoder_t *p_dec)
784 {
785     decoder_sys_t *p_sys = p_dec->p_sys;
786
787     for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
788         AndroidOpaquePicture_Release(p_sys->u.video.pp_inflight_pictures[i],
789                                      false);
790 }
791
792 static int InsertInflightPicture(decoder_t *p_dec, picture_sys_t *p_picsys)
793 {
794     decoder_sys_t *p_sys = p_dec->p_sys;
795
796     if (!p_picsys->priv.hw.p_dec)
797     {
798         p_picsys->priv.hw.p_dec = p_dec;
799         p_picsys->priv.hw.pf_release = ReleasePicture;
800         TAB_APPEND_CAST((picture_sys_t **),
801                         p_sys->u.video.i_inflight_pictures,
802                         p_sys->u.video.pp_inflight_pictures,
803                         p_picsys);
804     } /* else already attached */
805     return 0;
806 }
807
808 static void RemoveInflightPictures(decoder_t *p_dec)
809 {
810     decoder_sys_t *p_sys = p_dec->p_sys;
811
812     for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
813         AndroidOpaquePicture_DetachDecoder(p_sys->u.video.pp_inflight_pictures[i]);
814     TAB_CLEAN(p_sys->u.video.i_inflight_pictures,
815               p_sys->u.video.pp_inflight_pictures);
816 }
817
818 static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
819                                picture_t **pp_out_pic, block_t **pp_out_block)
820 {
821     decoder_sys_t *p_sys = p_dec->p_sys;
822     (void) pp_out_block;
823     assert(pp_out_pic);
824
825     if (p_out->type == MC_OUT_TYPE_BUF)
826     {
827         picture_t *p_pic = NULL;
828
829         /* Use the aspect ratio provided by the input (ie read from packetizer).
830          * Don't check the current value of the aspect ratio in fmt_out, since we
831          * want to allow changes in it to propagate. */
832         if (p_dec->fmt_in.video.i_sar_num != 0 && p_dec->fmt_in.video.i_sar_den != 0
833          && (p_dec->fmt_out.video.i_sar_num != p_dec->fmt_in.video.i_sar_num ||
834              p_dec->fmt_out.video.i_sar_den != p_dec->fmt_in.video.i_sar_den))
835         {
836             p_dec->fmt_out.video.i_sar_num = p_dec->fmt_in.video.i_sar_num;
837             p_dec->fmt_out.video.i_sar_den = p_dec->fmt_in.video.i_sar_den;
838             p_sys->b_update_format = true;
839         }
840
841         if (p_sys->b_update_format)
842         {
843             p_sys->b_update_format = false;
844             if (decoder_UpdateVideoFormat(p_dec) != 0)
845             {
846                 msg_Err(p_dec, "decoder_UpdateVideoFormat failed");
847                 p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
848                 return -1;
849             }
850         }
851
852         /* If the oldest input block had no PTS, the timestamp of
853          * the frame returned by MediaCodec might be wrong so we
854          * overwrite it with the corresponding dts. Call FifoGet
855          * first in order to avoid a gap if buffers are released
856          * due to an invalid format or a preroll */
857         int64_t forced_ts = timestamp_FifoGet(p_sys->u.video.timestamp_fifo);
858
859         if (!p_sys->b_has_format) {
860             msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
861             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
862         }
863
864         if (p_out->u.buf.i_ts <= p_sys->i_preroll_end)
865             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
866
867         if (!p_sys->api->b_direct_rendering && p_out->u.buf.p_ptr == NULL)
868         {
869             /* This can happen when receiving an EOS buffer */
870             msg_Warn(p_dec, "Invalid buffer, dropping frame");
871             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
872         }
873
874         p_pic = decoder_NewPicture(p_dec);
875         if (!p_pic) {
876             msg_Warn(p_dec, "NewPicture failed");
877             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
878         }
879
880         if (forced_ts == VLC_TS_INVALID)
881             p_pic->date = p_out->u.buf.i_ts;
882         else
883             p_pic->date = forced_ts;
884
885         if (p_sys->api->b_direct_rendering)
886         {
887             p_pic->p_sys->priv.hw.i_index = p_out->u.buf.i_index;
888             InsertInflightPicture(p_dec, p_pic->p_sys);
889         } else {
890             unsigned int chroma_div;
891             GetVlcChromaSizes(p_dec->fmt_out.i_codec,
892                               p_dec->fmt_out.video.i_width,
893                               p_dec->fmt_out.video.i_height,
894                               NULL, NULL, &chroma_div);
895             CopyOmxPicture(p_sys->u.video.i_pixel_format, p_pic,
896                            p_sys->u.video.i_slice_height, p_sys->u.video.i_stride,
897                            (uint8_t *)p_out->u.buf.p_ptr, chroma_div,
898                            &p_sys->u.video.ascd);
899
900             if (p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false))
901             {
902                 picture_Release(p_pic);
903                 return -1;
904             }
905         }
906         assert(!(*pp_out_pic));
907         *pp_out_pic = p_pic;
908         return 1;
909     } else {
910         assert(p_out->type == MC_OUT_TYPE_CONF);
911         p_sys->u.video.i_pixel_format = p_out->u.conf.video.pixel_format;
912         ArchitectureSpecificCopyHooksDestroy(p_sys->u.video.i_pixel_format,
913                                              &p_sys->u.video.ascd);
914
915         const char *name = "unknown";
916         if (p_sys->api->b_direct_rendering)
917             p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
918         else
919         {
920             if (!GetVlcChromaFormat(p_sys->u.video.i_pixel_format,
921                                     &p_dec->fmt_out.i_codec, &name)) {
922                 msg_Err(p_dec, "color-format not recognized");
923                 return -1;
924             }
925         }
926
927         msg_Err(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
928                 p_sys->u.video.i_pixel_format, name, p_out->u.conf.video.width, p_out->u.conf.video.height,
929                 p_out->u.conf.video.stride, p_out->u.conf.video.slice_height,
930                 p_out->u.conf.video.crop_left, p_out->u.conf.video.crop_top,
931                 p_out->u.conf.video.crop_right, p_out->u.conf.video.crop_bottom);
932
933         p_dec->fmt_out.video.i_width = p_out->u.conf.video.crop_right + 1 - p_out->u.conf.video.crop_left;
934         p_dec->fmt_out.video.i_height = p_out->u.conf.video.crop_bottom + 1 - p_out->u.conf.video.crop_top;
935         if (p_dec->fmt_out.video.i_width <= 1
936             || p_dec->fmt_out.video.i_height <= 1) {
937             p_dec->fmt_out.video.i_width = p_out->u.conf.video.width;
938             p_dec->fmt_out.video.i_height = p_out->u.conf.video.height;
939         }
940         p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
941         p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
942
943         p_sys->u.video.i_stride = p_out->u.conf.video.stride;
944         p_sys->u.video.i_slice_height = p_out->u.conf.video.slice_height;
945         if (p_sys->u.video.i_stride <= 0)
946             p_sys->u.video.i_stride = p_out->u.conf.video.width;
947         if (p_sys->u.video.i_slice_height <= 0)
948             p_sys->u.video.i_slice_height = p_out->u.conf.video.height;
949
950         ArchitectureSpecificCopyHooks(p_dec, p_out->u.conf.video.pixel_format,
951                                       p_out->u.conf.video.slice_height,
952                                       p_sys->u.video.i_stride, &p_sys->u.video.ascd);
953         if (p_sys->u.video.i_pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
954             p_sys->u.video.i_slice_height -= p_out->u.conf.video.crop_top/2;
955         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_IGNORE_PADDING))
956         {
957             p_sys->u.video.i_slice_height = 0;
958             p_sys->u.video.i_stride = p_dec->fmt_out.video.i_width;
959         }
960         p_sys->b_update_format = true;
961         p_sys->b_has_format = true;
962         return 0;
963     }
964 }
965
966 /* samples will be in the following order: FL FR FC LFE BL BR BC SL SR */
967 uint32_t pi_audio_order_src[] =
968 {
969     AOUT_CHAN_LEFT, AOUT_CHAN_RIGHT, AOUT_CHAN_CENTER, AOUT_CHAN_LFE,
970     AOUT_CHAN_REARLEFT, AOUT_CHAN_REARRIGHT, AOUT_CHAN_REARCENTER,
971     AOUT_CHAN_MIDDLELEFT, AOUT_CHAN_MIDDLERIGHT,
972 };
973
974 static int Audio_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
975                                picture_t **pp_out_pic, block_t **pp_out_block)
976 {
977     decoder_sys_t *p_sys = p_dec->p_sys;
978     (void) pp_out_pic;
979     assert(pp_out_block);
980
981     if (p_out->type == MC_OUT_TYPE_BUF)
982     {
983         block_t *p_block = NULL;
984         if (p_out->u.buf.p_ptr == NULL)
985         {
986             /* This can happen when receiving an EOS buffer */
987             msg_Warn(p_dec, "Invalid buffer, dropping frame");
988             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
989         }
990
991         if (!p_sys->b_has_format) {
992             msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
993             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
994         }
995
996         p_block = block_Alloc(p_out->u.buf.i_size);
997         if (!p_block)
998             return -1;
999         p_block->i_nb_samples = p_out->u.buf.i_size
1000                               / p_dec->fmt_out.audio.i_bytes_per_frame;
1001
1002         if (p_sys->u.audio.b_extract)
1003         {
1004             aout_ChannelExtract(p_block->p_buffer,
1005                                 p_dec->fmt_out.audio.i_channels,
1006                                 p_out->u.buf.p_ptr, p_sys->u.audio.i_channels,
1007                                 p_block->i_nb_samples, p_sys->u.audio.pi_extraction,
1008                                 p_dec->fmt_out.audio.i_bitspersample);
1009         }
1010         else
1011             memcpy(p_block->p_buffer, p_out->u.buf.p_ptr, p_out->u.buf.i_size);
1012
1013         if (p_out->u.buf.i_ts != 0 && p_out->u.buf.i_ts != date_Get(&p_sys->u.audio.i_end_date))
1014             date_Set(&p_sys->u.audio.i_end_date, p_out->u.buf.i_ts);
1015
1016         p_block->i_pts = date_Get(&p_sys->u.audio.i_end_date);
1017         p_block->i_length = date_Increment(&p_sys->u.audio.i_end_date,
1018                                            p_block->i_nb_samples)
1019                           - p_block->i_pts;
1020
1021         if (p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false))
1022         {
1023             block_Release(p_block);
1024             return -1;
1025         }
1026         *pp_out_block = p_block;
1027         return 1;
1028     } else {
1029         uint32_t i_layout_dst;
1030         int      i_channels_dst;
1031
1032         assert(p_out->type == MC_OUT_TYPE_CONF);
1033
1034         if (p_out->u.conf.audio.channel_count <= 0
1035          || p_out->u.conf.audio.channel_count > 8
1036          || p_out->u.conf.audio.sample_rate <= 0)
1037         {
1038             msg_Warn( p_dec, "invalid audio properties channels count %d, sample rate %d",
1039                       p_out->u.conf.audio.channel_count,
1040                       p_out->u.conf.audio.sample_rate);
1041             return -1;
1042         }
1043
1044         msg_Err(p_dec, "output: channel_count: %d, channel_mask: 0x%X, rate: %d",
1045                 p_out->u.conf.audio.channel_count, p_out->u.conf.audio.channel_mask,
1046                 p_out->u.conf.audio.sample_rate);
1047
1048         p_dec->fmt_out.i_codec = VLC_CODEC_S16N;
1049         p_dec->fmt_out.audio.i_format = p_dec->fmt_out.i_codec;
1050
1051         p_dec->fmt_out.audio.i_rate = p_out->u.conf.audio.sample_rate;
1052         date_Init(&p_sys->u.audio.i_end_date, p_out->u.conf.audio.sample_rate, 1 );
1053
1054         p_sys->u.audio.i_channels = p_out->u.conf.audio.channel_count;
1055         p_sys->u.audio.b_extract =
1056             aout_CheckChannelExtraction(p_sys->u.audio.pi_extraction,
1057                                         &i_layout_dst, &i_channels_dst,
1058                                         NULL, pi_audio_order_src,
1059                                         p_sys->u.audio.i_channels);
1060
1061         if (p_sys->u.audio.b_extract)
1062             msg_Warn(p_dec, "need channel extraction: %d -> %d",
1063                      p_sys->u.audio.i_channels, i_channels_dst);
1064
1065         p_dec->fmt_out.audio.i_original_channels =
1066         p_dec->fmt_out.audio.i_physical_channels = i_layout_dst;
1067         aout_FormatPrepare(&p_dec->fmt_out.audio);
1068
1069         if (decoder_UpdateAudioFormat(p_dec))
1070             return -1;
1071
1072         p_sys->b_has_format = true;
1073         return 0;
1074     }
1075 }
1076
1077 static void H264ProcessBlock(decoder_t *p_dec, block_t *p_block,
1078                              bool *p_csd_changed, bool *p_size_changed)
1079 {
1080     decoder_sys_t *p_sys = p_dec->p_sys;
1081
1082     assert(p_dec->fmt_in.i_codec == VLC_CODEC_H264 && p_block);
1083
1084     if (p_sys->u.video.i_nal_length_size)
1085     {
1086         h264_AVC_to_AnnexB(p_block->p_buffer, p_block->i_buffer,
1087                                p_sys->u.video.i_nal_length_size);
1088     } else if (H264SetCSD(p_dec, p_block->p_buffer, p_block->i_buffer,
1089                           p_size_changed) == VLC_SUCCESS)
1090     {
1091         *p_csd_changed = true;
1092     }
1093 }
1094
1095 static void HEVCProcessBlock(decoder_t *p_dec, block_t *p_block,
1096                              bool *p_csd_changed, bool *p_size_changed)
1097 {
1098     decoder_sys_t *p_sys = p_dec->p_sys;
1099
1100     assert(p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && p_block);
1101
1102     if (p_sys->u.video.i_nal_length_size)
1103     {
1104         h264_AVC_to_AnnexB(p_block->p_buffer, p_block->i_buffer,
1105                                p_sys->u.video.i_nal_length_size);
1106     }
1107
1108     /* TODO */
1109     VLC_UNUSED(p_csd_changed);
1110     VLC_UNUSED(p_size_changed);
1111 }
1112
1113 static void DecodeFlushLocked(decoder_t *p_dec)
1114 {
1115     decoder_sys_t *p_sys = p_dec->p_sys;
1116     bool b_had_input = p_sys->b_input_dequeued;
1117
1118     p_sys->b_input_dequeued = false;
1119     p_sys->b_flush_out = true;
1120     p_sys->i_preroll_end = 0;
1121     p_sys->b_output_ready = false;
1122     /* Resend CODEC_CONFIG buffer after a flush */
1123     p_sys->i_csd_send = 0;
1124
1125     p_sys->pf_on_flush(p_dec);
1126
1127     if (b_had_input && p_sys->api->flush(p_sys->api) != VLC_SUCCESS)
1128     {
1129         AbortDecoderLocked(p_dec);
1130         return;
1131     }
1132
1133     vlc_cond_broadcast(&p_sys->cond);
1134
1135     while (!p_sys->b_aborted && p_sys->b_flush_out)
1136         vlc_cond_wait(&p_sys->dec_cond, &p_sys->lock);
1137 }
1138
1139 static void DecodeFlush(decoder_t *p_dec)
1140 {
1141     decoder_sys_t *p_sys = p_dec->p_sys;
1142
1143     vlc_mutex_lock(&p_sys->lock);
1144     DecodeFlushLocked(p_dec);
1145     vlc_mutex_unlock(&p_sys->lock);
1146 }
1147
1148 static void *OutThread(void *data)
1149 {
1150     decoder_t *p_dec = data;
1151     decoder_sys_t *p_sys = p_dec->p_sys;
1152
1153     vlc_mutex_lock(&p_sys->lock);
1154     mutex_cleanup_push(&p_sys->lock);
1155     for (;;)
1156     {
1157         int i_index;
1158
1159         /* Wait for output ready */
1160         while (!p_sys->b_flush_out && !p_sys->b_output_ready)
1161             vlc_cond_wait(&p_sys->cond, &p_sys->lock);
1162
1163         if (p_sys->b_flush_out)
1164         {
1165             /* Acknowledge flushed state */
1166             p_sys->b_flush_out = false;
1167             vlc_cond_broadcast(&p_sys->dec_cond);
1168             continue;
1169         }
1170
1171         int canc = vlc_savecancel();
1172
1173         vlc_mutex_unlock(&p_sys->lock);
1174
1175         /* Wait for an output buffer. This function returns when a new output
1176          * is available or if output is flushed. */
1177         i_index = p_sys->api->dequeue_out(p_sys->api, -1);
1178
1179         vlc_mutex_lock(&p_sys->lock);
1180
1181         /* Ignore dequeue_out errors caused by flush */
1182         if (p_sys->b_flush_out)
1183         {
1184             /* If i_index >= 0, Release it. There is no way to know if i_index
1185              * is owned by us, so don't check the error. */
1186             if (i_index >= 0)
1187                 p_sys->api->release_out(p_sys->api, i_index, false);
1188
1189             /* Parse output format/buffers even when we are flushing */
1190             if (i_index != MC_API_INFO_OUTPUT_FORMAT_CHANGED
1191              && i_index != MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1192             {
1193                 vlc_restorecancel(canc);
1194                 continue;
1195             }
1196         }
1197
1198         /* Process output returned by dequeue_out */
1199         if (i_index >= 0 || i_index == MC_API_INFO_OUTPUT_FORMAT_CHANGED
1200          || i_index == MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1201         {
1202             struct mc_api_out out;
1203             int i_ret = p_sys->api->get_out(p_sys->api, i_index, &out);
1204
1205             if (i_ret == 1)
1206             {
1207                 picture_t *p_pic = NULL;
1208                 block_t *p_block = NULL;
1209
1210                 if (p_sys->pf_process_output(p_dec, &out, &p_pic,
1211                                              &p_block) == -1 && !out.b_eos)
1212                 {
1213                     msg_Err(p_dec, "pf_process_output failed");
1214                     vlc_restorecancel(canc);
1215                     break;
1216                 }
1217                 if (p_pic)
1218                     decoder_QueueVideo(p_dec, p_pic);
1219                 else if (p_block)
1220                     decoder_QueueAudio(p_dec, p_block);
1221
1222                 if (out.b_eos)
1223                 {
1224                     msg_Warn(p_dec, "EOS received, abort OutThread");
1225                     vlc_restorecancel(canc);
1226                     break;
1227                 }
1228             } else if (i_ret != 0)
1229             {
1230                 msg_Err(p_dec, "get_out failed");
1231                 vlc_restorecancel(canc);
1232                 break;
1233             }
1234         }
1235         else
1236         {
1237             msg_Err(p_dec, "dequeue_out failed");
1238             vlc_restorecancel(canc);
1239             break;
1240         }
1241         vlc_restorecancel(canc);
1242     }
1243     msg_Warn(p_dec, "OutThread stopped");
1244
1245     /* Signal DecoderFlush that the output thread aborted */
1246     p_sys->b_aborted = true;
1247     vlc_cond_signal(&p_sys->dec_cond);
1248
1249     vlc_cleanup_pop();
1250     vlc_mutex_unlock(&p_sys->lock);
1251
1252     return NULL;
1253 }
1254
1255 static block_t *GetNextBlock(decoder_sys_t *p_sys, block_t *p_block)
1256 {
1257     if (p_sys->i_csd_send < p_sys->i_csd_count)
1258         return p_sys->pp_csd[p_sys->i_csd_send++];
1259     else
1260         return p_block;
1261 }
1262
1263 /**
1264  * DecodeCommon called from DecodeVideo or DecodeAudio.
1265  * It returns -1 in case of error, 0 otherwise.
1266  */
1267 static int DecodeCommon(decoder_t *p_dec, block_t **pp_block)
1268 {
1269     decoder_sys_t *p_sys = p_dec->p_sys;
1270     int i_flags = 0;
1271     int i_ret;
1272     bool b_dequeue_timeout = false;
1273     bool b_draining;
1274
1275     if (pp_block != NULL && *pp_block == NULL)
1276         return 0;
1277
1278     vlc_mutex_lock(&p_sys->lock);
1279
1280     if (p_sys->b_aborted)
1281         goto end;
1282
1283     if (pp_block != NULL)
1284     {
1285         block_t *p_block = *pp_block;
1286         b_draining = false;
1287         if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED))
1288         {
1289             DecodeFlushLocked(p_dec);
1290             if (p_sys->b_aborted)
1291                 goto end;
1292             if (p_block->i_flags & BLOCK_FLAG_CORRUPTED)
1293                 goto end;
1294         }
1295
1296         /* Parse input block */
1297         if ((i_ret = p_sys->pf_on_new_block(p_dec, p_block, &i_flags)) == 1)
1298         {
1299             if (i_flags & (NEWBLOCK_FLAG_FLUSH|NEWBLOCK_FLAG_RESTART))
1300             {
1301                 msg_Warn(p_dec, "Flushing from DecodeCommon");
1302
1303                 /* Flush before restart to unblock OutThread */
1304                 DecodeFlushLocked(p_dec);
1305                 if (p_sys->b_aborted)
1306                     goto end;
1307
1308                 if (i_flags & NEWBLOCK_FLAG_RESTART)
1309                 {
1310                     msg_Warn(p_dec, "Restarting from DecodeCommon");
1311                     StopMediaCodec(p_dec);
1312                     if (StartMediaCodec(p_dec) != VLC_SUCCESS)
1313                     {
1314                         msg_Err(p_dec, "StartMediaCodec failed");
1315                         AbortDecoderLocked(p_dec);
1316                         goto end;
1317                     }
1318                 }
1319             }
1320         }
1321         else
1322         {
1323             if (i_ret != 0)
1324             {
1325                 AbortDecoderLocked(p_dec);
1326                 msg_Err(p_dec, "pf_on_new_block failed");
1327             }
1328             goto end;
1329         }
1330     }
1331     else
1332     {
1333         /* No input block, decoder is draining */
1334         msg_Err(p_dec, "Decoder is draining");
1335         b_draining = true;
1336
1337         if (!p_sys->b_output_ready )
1338         {
1339             /* Output no ready, no need to drain */
1340             goto end;
1341         }
1342     }
1343
1344     /* Abort if MediaCodec is not yet started */
1345     if (!p_sys->api->b_started)
1346         goto end;
1347
1348     /* Queue CSD blocks and input blocks */
1349     block_t *p_block = NULL;
1350     while (b_draining || (p_block = GetNextBlock(p_sys, *pp_block)))
1351     {
1352         int i_index;
1353
1354         vlc_mutex_unlock(&p_sys->lock);
1355         /* Wait for an input buffer. This function returns when a new input
1356          * buffer is available or after 1sec of timeout. */
1357         i_index = p_sys->api->dequeue_in(p_sys->api,
1358                                          p_sys->api->b_direct_rendering ?
1359                                          INT64_C(1000000) : -1);
1360         vlc_mutex_lock(&p_sys->lock);
1361
1362         if (p_sys->b_aborted)
1363             goto end;
1364
1365         bool b_config = false;
1366         mtime_t i_ts = 0;
1367         p_sys->b_input_dequeued = true;
1368         const void *p_buf = NULL;
1369         size_t i_size = 0;
1370
1371         if (i_index >= 0)
1372         {
1373             assert(b_draining || p_block != NULL);
1374             if (p_block != NULL)
1375             {
1376                 b_config = (p_block->i_flags & BLOCK_FLAG_CSD);
1377                 if (!b_config)
1378                 {
1379                     i_ts = p_block->i_pts;
1380                     if (!i_ts && p_block->i_dts)
1381                         i_ts = p_block->i_dts;
1382                 }
1383                 p_buf = p_block->p_buffer;
1384                 i_size = p_block->i_buffer;
1385             }
1386
1387             if (p_sys->api->queue_in(p_sys->api, i_index, p_buf, i_size,
1388                                      i_ts, b_config) == 0)
1389             {
1390                 if (!b_config && p_block != NULL)
1391                 {
1392                     if (p_block->i_flags & BLOCK_FLAG_PREROLL )
1393                         p_sys->i_preroll_end = i_ts;
1394
1395                     /* One input buffer is queued, signal OutThread that will
1396                      * fetch output buffers */
1397                     p_sys->b_output_ready = true;
1398                     vlc_cond_broadcast(&p_sys->cond);
1399
1400                     assert(p_block == *pp_block),
1401                     block_Release(p_block);
1402                     *pp_block = NULL;
1403                 }
1404                 b_dequeue_timeout = false;
1405                 if (b_draining)
1406                     break;
1407             } else
1408             {
1409                 msg_Err(p_dec, "queue_in failed");
1410                 AbortDecoderLocked(p_dec);
1411                 goto end;
1412             }
1413         }
1414         else if (i_index == MC_API_INFO_TRYAGAIN)
1415         {
1416             /* HACK: When direct rendering is enabled, there is a possible
1417              * deadlock between the Decoder and the Vout. It happens when the
1418              * Vout is paused and when the Decoder is flushing. In that case,
1419              * the Vout won't release any output buffers, therefore MediaCodec
1420              * won't dequeue any input buffers. To work around this issue,
1421              * release all output buffers if DecodeCommon is waiting more than
1422              * 1sec for a new input buffer. */
1423             if (!b_dequeue_timeout)
1424             {
1425                 msg_Warn(p_dec, "Decoder stuck: invalidate all buffers");
1426                 InvalidateAllPictures(p_dec);
1427                 b_dequeue_timeout = true;
1428                 continue;
1429             }
1430             else
1431             {
1432                 msg_Err(p_dec, "dequeue_in timeout: no input available for 2secs");
1433                 AbortDecoderLocked(p_dec);
1434                 goto end;
1435             }
1436         }
1437         else
1438         {
1439             msg_Err(p_dec, "dequeue_in failed");
1440             AbortDecoderLocked(p_dec);
1441             goto end;
1442         }
1443     }
1444
1445     if (b_draining)
1446     {
1447         msg_Warn(p_dec, "EOS sent, waiting for OutThread");
1448
1449         /* Wait for the OutThread to stop (and process all remaining output
1450          * frames. Use a timeout here since we can't know if all decoders will
1451          * behave correctly. */
1452         mtime_t deadline = mdate() + INT64_C(1000000);
1453         while (!p_sys->b_aborted
1454             && vlc_cond_timedwait(&p_sys->dec_cond, &p_sys->lock, deadline) == 0);
1455
1456         if (!p_sys->b_aborted)
1457             msg_Err(p_dec, "OutThread timed out");
1458
1459         /* In case pf_decode is called again (it shouldn't happen) */
1460         p_sys->b_error_signaled = true;
1461
1462         vlc_mutex_unlock(&p_sys->lock);
1463         return 0;
1464     }
1465
1466 end:
1467     if (pp_block && *pp_block)
1468     {
1469         block_Release(*pp_block);
1470         *pp_block = NULL;
1471     }
1472     if (p_sys->b_aborted)
1473     {
1474         if (!p_sys->b_error_signaled) {
1475             /* Signal the error to the Java.
1476              * TODO: remove this when there is a decoder fallback */
1477             if (p_dec->fmt_in.i_cat == VIDEO_ES)
1478                 AWindowHandler_sendHardwareAccelerationError(VLC_OBJECT(p_dec),
1479                                                              p_sys->u.video.p_awh);
1480             p_sys->b_error_signaled = true;
1481             vlc_cond_broadcast(&p_sys->cond);
1482         }
1483         vlc_mutex_unlock(&p_sys->lock);
1484         return -1;
1485     }
1486     else
1487     {
1488         vlc_mutex_unlock(&p_sys->lock);
1489         return 0;
1490     }
1491 }
1492
1493 static int Video_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
1494 {
1495     decoder_sys_t *p_sys = p_dec->p_sys;
1496     bool b_csd_changed = false, b_size_changed = false;
1497
1498     if (p_block->i_flags & BLOCK_FLAG_INTERLACED_MASK
1499         && !p_sys->api->b_support_interlaced)
1500         return -1;
1501
1502     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
1503         H264ProcessBlock(p_dec, p_block, &b_csd_changed, &b_size_changed);
1504     else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
1505         HEVCProcessBlock(p_dec, p_block, &b_csd_changed, &b_size_changed);
1506
1507     if (b_csd_changed)
1508     {
1509         if (b_size_changed || !p_sys->api->b_started)
1510         {
1511             if (p_sys->api->b_started)
1512                 msg_Err(p_dec, "SPS/PPS changed during playback and "
1513                         "video size are different. Restart it !");
1514             *p_flags |= NEWBLOCK_FLAG_RESTART;
1515         } else
1516         {
1517             msg_Err(p_dec, "SPS/PPS changed during playback. Flush it");
1518             *p_flags |= NEWBLOCK_FLAG_FLUSH;
1519         }
1520     }
1521
1522     if (!p_sys->api->b_started)
1523     {
1524         *p_flags |= NEWBLOCK_FLAG_RESTART;
1525
1526         /* Don't start if we don't have any csd */
1527         if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
1528          && !p_dec->fmt_in.i_extra && !p_sys->pp_csd)
1529             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1530
1531         /* Don't start if we don't have a valid video size */
1532         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_NEED_SIZE)
1533          && (!p_sys->u.video.i_width || !p_sys->u.video.i_height))
1534             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1535     }
1536
1537     timestamp_FifoPut(p_sys->u.video.timestamp_fifo,
1538                       p_block->i_pts ? VLC_TS_INVALID : p_block->i_dts);
1539
1540     return 1;
1541 }
1542
1543 static void Video_OnFlush(decoder_t *p_dec)
1544 {
1545     decoder_sys_t *p_sys = p_dec->p_sys;
1546
1547     timestamp_FifoEmpty(p_sys->u.video.timestamp_fifo);
1548     /* Invalidate all pictures that are currently in flight
1549      * since flushing make all previous indices returned by
1550      * MediaCodec invalid. */
1551     if (p_sys->api->b_direct_rendering)
1552         InvalidateAllPictures(p_dec);
1553 }
1554
1555 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
1556 {
1557     DecodeCommon(p_dec, pp_block);
1558     return NULL;
1559 }
1560
1561 static int Audio_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
1562 {
1563     decoder_sys_t *p_sys = p_dec->p_sys;
1564
1565     /* We've just started the stream, wait for the first PTS. */
1566     if (!date_Get(&p_sys->u.audio.i_end_date))
1567     {
1568         if (p_block->i_pts <= VLC_TS_INVALID)
1569             return 0;
1570         date_Set(&p_sys->u.audio.i_end_date, p_block->i_pts);
1571     }
1572
1573     /* try delayed opening if there is a new extra data */
1574     if (!p_sys->api->b_started)
1575     {
1576         p_dec->p_sys->u.audio.i_channels = p_dec->fmt_in.audio.i_channels;
1577
1578         *p_flags |= NEWBLOCK_FLAG_RESTART;
1579
1580         /* Don't start if we don't have any csd */
1581         if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
1582          && !p_dec->fmt_in.i_extra)
1583             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1584
1585         /* Don't start if we don't have a valid channels count */
1586         if ((p_sys->i_quirks & OMXCODEC_AUDIO_QUIRKS_NEED_CHANNELS)
1587          && !p_dec->p_sys->u.audio.i_channels)
1588             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1589     }
1590     return 1;
1591 }
1592
1593 static void Audio_OnFlush(decoder_t *p_dec)
1594 {
1595     decoder_sys_t *p_sys = p_dec->p_sys;
1596
1597     date_Set(&p_sys->u.audio.i_end_date, VLC_TS_INVALID);
1598 }
1599
1600 static block_t *DecodeAudio(decoder_t *p_dec, block_t **pp_block)
1601 {
1602     DecodeCommon(p_dec, pp_block);
1603     return NULL;
1604 }