mediacodec: remove ArchitectureSpecificCopyData
[vlc.git] / modules / codec / omxil / mediacodec.c
1 /*****************************************************************************
2  * mediacodec.c: Video decoder module using the Android MediaCodec API
3  *****************************************************************************
4  * Copyright (C) 2012 Martin Storsjo
5  *
6  * Authors: Martin Storsjo <martin@martin.st>
7  *
8  * This program is free software; you can redistribute it and/or modify it
9  * under the terms of the GNU Lesser General Public License as published by
10  * the Free Software Foundation; either version 2.1 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16  * GNU Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21  *****************************************************************************/
22
23 /*****************************************************************************
24  * Preamble
25  *****************************************************************************/
26 #ifdef HAVE_CONFIG_H
27 # include "config.h"
28 #endif
29
30 #include <jni.h>
31 #include <stdint.h>
32 #include <assert.h>
33
34 #include <vlc_common.h>
35 #include <vlc_aout.h>
36 #include <vlc_plugin.h>
37 #include <vlc_codec.h>
38 #include <vlc_block_helper.h>
39 #include <vlc_cpu.h>
40 #include <vlc_memory.h>
41 #include <vlc_timestamp_helper.h>
42 #include <vlc_threads.h>
43
44 #include "mediacodec.h"
45 #include "../../packetizer/h264_nal.h"
46 #include "../../packetizer/hevc_nal.h"
47 #include "../../packetizer/hxxx_nal.h"
48 #include <OMX_Core.h>
49 #include <OMX_Component.h>
50 #include "omxil_utils.h"
51 #include "../../video_output/android/android_window.h"
52
53 #define BLOCK_FLAG_CSD (0x01 << BLOCK_FLAG_PRIVATE_SHIFT)
54
55 /* Codec Specific Data */
56 struct csd
57 {
58     uint8_t *p_buf;
59     size_t i_size;
60 };
61
62 #define NEWBLOCK_FLAG_RESTART (0x01)
63 #define NEWBLOCK_FLAG_FLUSH (0x02)
64 /**
65  * Callback called when a new block is processed from DecodeCommon.
66  * It returns -1 in case of error, 0 if block should be dropped, 1 otherwise.
67  */
68 typedef int (*dec_on_new_block_cb)(decoder_t *, block_t *, int *);
69
70 /**
71  * Callback called when decoder is flushing.
72  */
73 typedef void (*dec_on_flush_cb)(decoder_t *);
74
75 /**
76  * Callback called when DecodeCommon try to get an output buffer (pic or block).
77  * It returns -1 in case of error, or the number of output buffer returned.
78  */
79 typedef int (*dec_process_output_cb)(decoder_t *, mc_api_out *, picture_t **, block_t **);
80
81 struct decoder_sys_t
82 {
83     mc_api *api;
84
85     /* Codec Specific Data buffer: sent in DecodeCommon after a start or a flush
86      * with the BUFFER_FLAG_CODEC_CONFIG flag.*/
87     block_t **pp_csd;
88     size_t i_csd_count;
89     size_t i_csd_send;
90
91     bool b_update_format;
92     bool b_has_format;
93
94     int64_t i_preroll_end;
95     int     i_quirks;
96
97     /* Specific Audio/Video callbacks */
98     dec_on_new_block_cb     pf_on_new_block;
99     dec_on_flush_cb         pf_on_flush;
100     dec_process_output_cb   pf_process_output;
101
102     vlc_mutex_t     lock;
103     vlc_thread_t    out_thread;
104     /* Cond used to signal the output thread */
105     vlc_cond_t      cond;
106     /* Cond used to signal the decoder thread */
107     vlc_cond_t      dec_cond;
108     /* Set to true by pf_flush to signal the output thread to flush */
109     bool            b_flush_out;
110     /* If true, the output thread will start to dequeue output pictures */
111     bool            b_output_ready;
112     /* If true, the first input block was successfully dequeued */
113     bool            b_input_dequeued;
114     bool            b_aborted;
115     /* TODO: remove */
116     bool            b_error_signaled;
117
118     union
119     {
120         struct
121         {
122             AWindowHandler *p_awh;
123             unsigned int i_stride, i_slice_height, i_width, i_height;
124             int i_pixel_format;
125             uint8_t i_nal_length_size;
126             size_t i_h264_profile;
127             /* stores the inflight picture for each output buffer or NULL */
128             picture_sys_t** pp_inflight_pictures;
129             unsigned int i_inflight_pictures;
130             timestamp_fifo_t *timestamp_fifo;
131         } video;
132         struct {
133             date_t i_end_date;
134             int i_channels;
135             bool b_extract;
136             /* Some audio decoders need a valid channel count */
137             bool b_need_channels;
138             int pi_extraction[AOUT_CHAN_MAX];
139         } audio;
140     } u;
141 };
142
143 /*****************************************************************************
144  * Local prototypes
145  *****************************************************************************/
146 static int  OpenDecoderJni(vlc_object_t *);
147 static int  OpenDecoderNdk(vlc_object_t *);
148 static void CleanDecoder(decoder_t *);
149 static void CloseDecoder(vlc_object_t *);
150
151 static int Video_OnNewBlock(decoder_t *, block_t *, int *);
152 static void Video_OnFlush(decoder_t *);
153 static int Video_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
154 static picture_t *DecodeVideo(decoder_t *, block_t **);
155
156 static int Audio_OnNewBlock(decoder_t *, block_t *, int *);
157 static void Audio_OnFlush(decoder_t *);
158 static int Audio_ProcessOutput(decoder_t *, mc_api_out *, picture_t **, block_t **);
159 static block_t *DecodeAudio(decoder_t *, block_t **);
160
161 static void DecodeFlushLocked(decoder_t *);
162 static void DecodeFlush(decoder_t *);
163 static void StopMediaCodec(decoder_t *);
164 static void *OutThread(void *);
165
166 static void InvalidateAllPictures(decoder_t *);
167 static void RemoveInflightPictures(decoder_t *);
168
169 /*****************************************************************************
170  * Module descriptor
171  *****************************************************************************/
172 #define DIRECTRENDERING_TEXT N_("Android direct rendering")
173 #define DIRECTRENDERING_LONGTEXT N_(\
174         "Enable Android direct rendering using opaque buffers.")
175
176 #define MEDIACODEC_AUDIO_TEXT "Use MediaCodec for audio decoding"
177 #define MEDIACODEC_AUDIO_LONGTEXT "Still experimental."
178
179 #define MEDIACODEC_TUNNELEDPLAYBACK_TEXT "Use a tunneled surface for playback"
180
181 #define CFG_PREFIX "mediacodec-"
182
183 vlc_module_begin ()
184     set_description( N_("Video decoder using Android MediaCodec via NDK") )
185     set_category( CAT_INPUT )
186     set_subcategory( SUBCAT_INPUT_VCODEC )
187     set_section( N_("Decoding") , NULL )
188     set_capability( "decoder", 0 ) /* Only enabled via commandline arguments */
189     add_bool(CFG_PREFIX "dr", true,
190              DIRECTRENDERING_TEXT, DIRECTRENDERING_LONGTEXT, true)
191     add_bool(CFG_PREFIX "audio", false,
192              MEDIACODEC_AUDIO_TEXT, MEDIACODEC_AUDIO_LONGTEXT, true)
193     add_bool(CFG_PREFIX "tunneled-playback", false,
194              MEDIACODEC_TUNNELEDPLAYBACK_TEXT, NULL, true)
195     set_callbacks( OpenDecoderNdk, CloseDecoder )
196     add_shortcut( "mediacodec_ndk" )
197     add_submodule ()
198         set_description( N_("Video decoder using Android MediaCodec via JNI") )
199         set_capability( "decoder", 0 )
200         set_callbacks( OpenDecoderJni, CloseDecoder )
201         add_shortcut( "mediacodec_jni" )
202 vlc_module_end ()
203
204 static void CSDFree(decoder_t *p_dec)
205 {
206     decoder_sys_t *p_sys = p_dec->p_sys;
207
208     if (p_sys->pp_csd)
209     {
210         for (unsigned int i = 0; i < p_sys->i_csd_count; ++i)
211             block_Release(p_sys->pp_csd[i]);
212         free(p_sys->pp_csd);
213         p_sys->pp_csd = NULL;
214     }
215     p_sys->i_csd_count = 0;
216 }
217
218 /* Create the p_sys->p_csd that will be sent from DecodeCommon */
219 static int CSDDup(decoder_t *p_dec, const struct csd *p_csd, size_t i_count)
220 {
221     decoder_sys_t *p_sys = p_dec->p_sys;
222
223     CSDFree(p_dec);
224
225     p_sys->pp_csd = malloc(i_count * sizeof(block_t *));
226     if (!p_sys->pp_csd)
227         return VLC_ENOMEM;
228
229     for (size_t i = 0; i < i_count; ++i)
230     {
231         p_sys->pp_csd[i] = block_Alloc(p_csd[i].i_size);
232         if (!p_sys->pp_csd[i])
233         {
234             CSDFree(p_dec);
235             return VLC_ENOMEM;
236         }
237         p_sys->pp_csd[i]->i_flags = BLOCK_FLAG_CSD;
238         memcpy(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf, p_csd[i].i_size);
239         p_sys->i_csd_count++;
240     }
241
242     p_sys->i_csd_send = 0;
243     return VLC_SUCCESS;
244 }
245
246 static bool CSDCmp(decoder_t *p_dec, struct csd *p_csd, size_t i_csd_count)
247 {
248     decoder_sys_t *p_sys = p_dec->p_sys;
249
250     if (p_sys->i_csd_count != i_csd_count)
251         return false;
252     for (size_t i = 0; i < i_csd_count; ++i)
253     {
254         if (p_sys->pp_csd[i]->i_buffer != p_csd[i].i_size
255          || memcmp(p_sys->pp_csd[i]->p_buffer, p_csd[i].p_buf,
256                    p_csd[i].i_size) != 0)
257             return false;
258     }
259     return true;
260 }
261
262 /* Fill the p_sys->p_csd struct with H264 Parameter Sets */
263 static int H264SetCSD(decoder_t *p_dec, void *p_buf, size_t i_size,
264                       bool *p_size_changed)
265 {
266     decoder_sys_t *p_sys = p_dec->p_sys;
267     uint8_t *p_sps_buf = NULL, *p_pps_buf = NULL;
268     size_t i_sps_size = 0, i_pps_size = 0;
269
270     /* Check if p_buf contains a valid SPS PPS */
271     if (h264_get_spspps(p_buf, i_size, &p_sps_buf, &i_sps_size,
272                         &p_pps_buf, &i_pps_size) == 0 )
273     {
274         struct csd csd[2];
275         int i_csd_count = 0;
276
277         const uint8_t *p_buffer = p_sps_buf;
278         size_t i_buffer = i_sps_size;
279         if(!hxxx_strip_AnnexB_startcode(&p_buffer, &i_buffer))
280             return VLC_EGENERIC;
281
282         h264_sequence_parameter_set_t *p_sps = h264_decode_sps(p_buffer, i_buffer, true);
283         if( !p_sps )
284             return VLC_EGENERIC;
285
286         unsigned vsize[4];
287         (void) h264_get_picture_size( p_sps, &vsize[0], &vsize[1], &vsize[2], &vsize[3] );
288         /* FIXME: what to do with visible width/height ? */
289
290         if (i_sps_size)
291         {
292             csd[i_csd_count].p_buf = p_sps_buf;
293             csd[i_csd_count].i_size = i_sps_size;
294             i_csd_count++;
295         }
296         if (i_pps_size)
297         {
298             csd[i_csd_count].p_buf = p_pps_buf;
299             csd[i_csd_count].i_size = i_pps_size;
300             i_csd_count++;
301         }
302
303         /* Compare the SPS PPS with the old one */
304         if (!CSDCmp(p_dec, csd, i_csd_count))
305         {
306             msg_Warn(p_dec, "New SPS/PPS found, id: %d size: %ux%u sps: %d pps: %d",
307                      p_sps->i_id, vsize[0], vsize[1],
308                      i_sps_size, i_pps_size);
309
310             /* In most use cases, p_sys->p_csd[0] contains a SPS, and
311              * p_sys->p_csd[1] contains a PPS */
312             if (CSDDup(p_dec, csd, i_csd_count))
313             {
314                 h264_release_sps( p_sps );
315                 return VLC_ENOMEM;
316             }
317
318             if (p_size_changed)
319                 *p_size_changed = (vsize[0] != p_sys->u.video.i_width
320                                 || vsize[1] != p_sys->u.video.i_height);
321
322             p_sys->u.video.i_width = vsize[0];
323             p_sys->u.video.i_height = vsize[1];
324
325             h264_release_sps( p_sps );
326
327             return VLC_SUCCESS;
328         }
329
330         h264_release_sps( p_sps );
331     }
332
333     return VLC_EGENERIC;
334 }
335
336 static int ParseVideoExtra(decoder_t *p_dec, uint8_t *p_extra, int i_extra)
337 {
338     decoder_sys_t *p_sys = p_dec->p_sys;
339
340     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
341      || p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
342     {
343         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
344         {
345             if (h264_isavcC(p_extra, i_extra))
346             {
347                 size_t i_size = 0;
348                 uint8_t *p_buf = h264_avcC_to_AnnexB_NAL(p_extra, i_extra, &i_size,
349                                                          &p_sys->u.video.i_nal_length_size);
350                 if(p_buf)
351                 {
352                     H264SetCSD(p_dec, p_buf, i_size, NULL);
353                     free(p_buf);
354                 }
355             }
356             else
357                 H264SetCSD(p_dec, p_extra, i_extra, NULL);
358         }
359         else  /* FIXME and refactor: CSDDup vs CSDless SetCSD */
360         {
361             if (hevc_ishvcC(p_extra, i_extra))
362             {
363                 struct csd csd;
364                 csd.p_buf = hevc_hvcC_to_AnnexB_NAL(p_extra, i_extra, &csd.i_size,
365                                                     &p_sys->u.video.i_nal_length_size);
366                 if(csd.p_buf)
367                 {
368                     CSDDup(p_dec, &csd, 1);
369                     free(csd.p_buf);
370                 }
371             }
372             /* FIXME: what to do with AnnexB ? */
373         }
374     }
375     return VLC_SUCCESS;
376 }
377
378 /*****************************************************************************
379  * StartMediaCodec: Create the mediacodec instance
380  *****************************************************************************/
381 static int StartMediaCodec(decoder_t *p_dec)
382 {
383     decoder_sys_t *p_sys = p_dec->p_sys;
384     int i_ret = 0;
385     union mc_api_args args;
386
387     if (p_dec->fmt_in.i_extra && !p_sys->pp_csd)
388     {
389         /* Try first to configure specific Video CSD */
390         if (p_dec->fmt_in.i_cat == VIDEO_ES)
391             i_ret = ParseVideoExtra(p_dec, p_dec->fmt_in.p_extra,
392                                     p_dec->fmt_in.i_extra);
393
394         if (i_ret != VLC_SUCCESS)
395             return i_ret;
396
397         /* Set default CSD if ParseVideoExtra failed to configure one */
398         if (!p_sys->pp_csd)
399         {
400             struct csd csd;
401
402             csd.p_buf = p_dec->fmt_in.p_extra;
403             csd.i_size = p_dec->fmt_in.i_extra;
404             CSDDup(p_dec, &csd, 1);
405         }
406     }
407
408     if (p_dec->fmt_in.i_cat == VIDEO_ES)
409     {
410         if (!p_sys->u.video.i_width || !p_sys->u.video.i_height)
411         {
412             msg_Err(p_dec, "invalid size, abort MediaCodec");
413             return VLC_EGENERIC;
414         }
415         args.video.i_width = p_sys->u.video.i_width;
416         args.video.i_height = p_sys->u.video.i_height;
417
418         switch (p_dec->fmt_in.video.orientation)
419         {
420             case ORIENT_ROTATED_90:
421                 args.video.i_angle = 90;
422                 break;
423             case ORIENT_ROTATED_180:
424                 args.video.i_angle = 180;
425                 break;
426             case ORIENT_ROTATED_270:
427                 args.video.i_angle = 270;
428                 break;
429             default:
430                 args.video.i_angle = 0;
431         }
432
433         /* Check again the codec name if h264 profile changed */
434         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264
435          && !p_sys->u.video.i_h264_profile)
436         {
437             h264_get_profile_level(&p_dec->fmt_in,
438                                    &p_sys->u.video.i_h264_profile, NULL, NULL);
439             if (p_sys->u.video.i_h264_profile)
440             {
441                 free(p_sys->api->psz_name);
442                 p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
443                                                           p_sys->api->psz_mime,
444                                                           p_sys->u.video.i_h264_profile);
445                 if (!p_sys->api->psz_name)
446                     return VLC_EGENERIC;
447             }
448         }
449
450         if (!p_sys->u.video.p_awh && var_InheritBool(p_dec, CFG_PREFIX "dr"))
451         {
452             if ((p_sys->u.video.p_awh = AWindowHandler_new(VLC_OBJECT(p_dec))))
453             {
454                 /* Direct rendering:
455                  * The surface must be released by the Vout before calling
456                  * start. Request a valid OPAQUE Vout to release any non-OPAQUE
457                  * Vout that will release the surface.
458                  */
459                 p_dec->fmt_out.video.i_width = p_sys->u.video.i_width;
460                 p_dec->fmt_out.video.i_height = p_sys->u.video.i_height;
461                 p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
462                 if (decoder_UpdateVideoFormat(p_dec) != 0)
463                 {
464                     msg_Err(p_dec, "Opaque Vout request failed: "
465                                    "fallback to non opaque");
466
467                     AWindowHandler_destroy(p_sys->u.video.p_awh);
468                     p_sys->u.video.p_awh = NULL;
469                 }
470             }
471         }
472         args.video.p_awh = p_sys->u.video.p_awh;
473         args.video.b_tunneled_playback = args.video.p_awh ?
474                 var_InheritBool(p_dec, CFG_PREFIX "tunneled-playback") : false;
475     }
476     else
477     {
478         date_Set(&p_sys->u.audio.i_end_date, VLC_TS_INVALID);
479
480         args.audio.i_sample_rate    = p_dec->fmt_in.audio.i_rate;
481         args.audio.i_channel_count  = p_dec->p_sys->u.audio.i_channels;
482     }
483
484     return p_sys->api->start(p_sys->api, &args);
485 }
486
487 /*****************************************************************************
488  * StopMediaCodec: Close the mediacodec instance
489  *****************************************************************************/
490 static void StopMediaCodec(decoder_t *p_dec)
491 {
492     decoder_sys_t *p_sys = p_dec->p_sys;
493
494     /* Remove all pictures that are currently in flight in order
495      * to prevent the vout from using destroyed output buffers. */
496     if (p_sys->api->b_direct_rendering)
497         RemoveInflightPictures(p_dec);
498
499     p_sys->api->stop(p_sys->api);
500     if (p_dec->fmt_in.i_cat == VIDEO_ES && p_sys->u.video.p_awh)
501         AWindowHandler_releaseSurface(p_sys->u.video.p_awh, AWindow_Video);
502 }
503
504 /*****************************************************************************
505  * OpenDecoder: Create the decoder instance
506  *****************************************************************************/
507 static int OpenDecoder(vlc_object_t *p_this, pf_MediaCodecApi_init pf_init)
508 {
509     decoder_t *p_dec = (decoder_t *)p_this;
510     decoder_sys_t *p_sys;
511     mc_api *api;
512     const char *mime = NULL;
513     bool b_late_opening = false;
514
515     /* Video or Audio if "mediacodec-audio" bool is true */
516     if (p_dec->fmt_in.i_cat != VIDEO_ES && (p_dec->fmt_in.i_cat != AUDIO_ES
517      || !var_InheritBool(p_dec, CFG_PREFIX "audio")))
518         return VLC_EGENERIC;
519
520     if (p_dec->fmt_in.i_cat == VIDEO_ES)
521     {
522         if (!p_dec->fmt_in.video.i_width || !p_dec->fmt_in.video.i_height)
523         {
524             /* We can handle h264 without a valid video size */
525             if (p_dec->fmt_in.i_codec != VLC_CODEC_H264)
526             {
527                 msg_Dbg(p_dec, "resolution (%dx%d) not supported",
528                         p_dec->fmt_in.video.i_width, p_dec->fmt_in.video.i_height);
529                 return VLC_EGENERIC;
530             }
531         }
532
533         switch (p_dec->fmt_in.i_codec) {
534         case VLC_CODEC_HEVC: mime = "video/hevc"; break;
535         case VLC_CODEC_H264: mime = "video/avc"; break;
536         case VLC_CODEC_H263: mime = "video/3gpp"; break;
537         case VLC_CODEC_MP4V: mime = "video/mp4v-es"; break;
538         case VLC_CODEC_WMV3: mime = "video/x-ms-wmv"; break;
539         case VLC_CODEC_VC1:  mime = "video/wvc1"; break;
540         case VLC_CODEC_VP8:  mime = "video/x-vnd.on2.vp8"; break;
541         case VLC_CODEC_VP9:  mime = "video/x-vnd.on2.vp9"; break;
542         /* case VLC_CODEC_MPGV: mime = "video/mpeg2"; break; */
543         }
544     }
545     else
546     {
547         switch (p_dec->fmt_in.i_codec) {
548         case VLC_CODEC_AMR_NB: mime = "audio/3gpp"; break;
549         case VLC_CODEC_AMR_WB: mime = "audio/amr-wb"; break;
550         case VLC_CODEC_MPGA:
551         case VLC_CODEC_MP3:    mime = "audio/mpeg"; break;
552         case VLC_CODEC_MP2:    mime = "audio/mpeg-L2"; break;
553         case VLC_CODEC_MP4A:   mime = "audio/mp4a-latm"; break;
554         case VLC_CODEC_QCELP:  mime = "audio/qcelp"; break;
555         case VLC_CODEC_VORBIS: mime = "audio/vorbis"; break;
556         case VLC_CODEC_OPUS:   mime = "audio/opus"; break;
557         case VLC_CODEC_ALAW:   mime = "audio/g711-alaw"; break;
558         case VLC_CODEC_MULAW:  mime = "audio/g711-mlaw"; break;
559         case VLC_CODEC_FLAC:   mime = "audio/flac"; break;
560         case VLC_CODEC_GSM:    mime = "audio/gsm"; break;
561         case VLC_CODEC_A52:    mime = "audio/ac3"; break;
562         case VLC_CODEC_EAC3:   mime = "audio/eac3"; break;
563         case VLC_CODEC_ALAC:   mime = "audio/alac"; break;
564         case VLC_CODEC_DTS:    mime = "audio/vnd.dts"; break;
565         /* case VLC_CODEC_: mime = "audio/mpeg-L1"; break; */
566         /* case VLC_CODEC_: mime = "audio/aac-adts"; break; */
567         }
568     }
569     if (!mime)
570     {
571         msg_Dbg(p_dec, "codec %4.4s not supported",
572                 (char *)&p_dec->fmt_in.i_codec);
573         return VLC_EGENERIC;
574     }
575
576     api = calloc(1, sizeof(mc_api));
577     if (!api)
578         return VLC_ENOMEM;
579     api->p_obj = p_this;
580     api->b_video = p_dec->fmt_in.i_cat == VIDEO_ES;
581     if (pf_init(api) != VLC_SUCCESS)
582     {
583         free(api);
584         return VLC_EGENERIC;
585     }
586
587     /* Allocate the memory needed to store the decoder's structure */
588     if ((p_sys = calloc(1, sizeof(*p_sys))) == NULL)
589     {
590         api->clean(api);
591         free(api);
592         return VLC_ENOMEM;
593     }
594     p_sys->api = api;
595     p_dec->p_sys = p_sys;
596
597     p_dec->pf_decode_video = DecodeVideo;
598     p_dec->pf_decode_audio = DecodeAudio;
599     p_dec->pf_flush        = DecodeFlush;
600
601     p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
602     p_dec->fmt_out.video = p_dec->fmt_in.video;
603     p_dec->fmt_out.audio = p_dec->fmt_in.audio;
604     p_sys->api->psz_mime = mime;
605
606     vlc_mutex_init(&p_sys->lock);
607     vlc_cond_init(&p_sys->cond);
608     vlc_cond_init(&p_sys->dec_cond);
609
610     if (p_dec->fmt_in.i_cat == VIDEO_ES)
611     {
612         p_sys->pf_on_new_block = Video_OnNewBlock;
613         p_sys->pf_on_flush = Video_OnFlush;
614         p_sys->pf_process_output = Video_ProcessOutput;
615         p_sys->u.video.i_width = p_dec->fmt_in.video.i_width;
616         p_sys->u.video.i_height = p_dec->fmt_in.video.i_height;
617
618         p_sys->u.video.timestamp_fifo = timestamp_FifoNew(32);
619         if (!p_sys->u.video.timestamp_fifo)
620             goto bailout;
621         TAB_INIT( p_sys->u.video.i_inflight_pictures,
622                   p_sys->u.video.pp_inflight_pictures );
623
624         if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
625             h264_get_profile_level(&p_dec->fmt_in,
626                                    &p_sys->u.video.i_h264_profile, NULL, NULL);
627
628         p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
629                                                   p_sys->api->psz_mime,
630                                                   p_sys->u.video.i_h264_profile);
631         if (!p_sys->api->psz_name)
632             goto bailout;
633
634         p_sys->i_quirks = OMXCodec_GetQuirks(VIDEO_ES,
635                                              p_dec->fmt_in.i_codec,
636                                              p_sys->api->psz_name,
637                                              strlen(p_sys->api->psz_name));
638
639         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_NEED_SIZE)
640          && (!p_sys->u.video.i_width || !p_sys->u.video.i_height))
641         {
642             msg_Warn(p_dec, "waiting for a valid video size for codec %4.4s",
643                      (const char *)&p_dec->fmt_in.i_codec);
644             b_late_opening = true;
645         }
646     }
647     else
648     {
649         p_sys->pf_on_new_block = Audio_OnNewBlock;
650         p_sys->pf_on_flush = Audio_OnFlush;
651         p_sys->pf_process_output = Audio_ProcessOutput;
652         p_sys->u.audio.i_channels = p_dec->fmt_in.audio.i_channels;
653
654         p_sys->api->psz_name = MediaCodec_GetName(VLC_OBJECT(p_dec),
655                                                   p_sys->api->psz_mime, 0);
656         if (!p_sys->api->psz_name)
657             goto bailout;
658
659         p_sys->i_quirks = OMXCodec_GetQuirks(AUDIO_ES,
660                                              p_dec->fmt_in.i_codec,
661                                              p_sys->api->psz_name,
662                                              strlen(p_sys->api->psz_name));
663         if ((p_sys->i_quirks & OMXCODEC_AUDIO_QUIRKS_NEED_CHANNELS)
664          && !p_sys->u.audio.i_channels)
665         {
666             msg_Warn(p_dec, "waiting for valid channel count");
667             b_late_opening = true;
668         }
669     }
670     if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
671      && !p_dec->fmt_in.i_extra)
672     {
673         msg_Warn(p_dec, "waiting for extra data for codec %4.4s",
674                  (const char *)&p_dec->fmt_in.i_codec);
675         if (p_dec->fmt_in.i_codec == VLC_CODEC_MP4V)
676         {
677             msg_Warn(p_dec, "late opening with MPEG4 not handled"); /* TODO */
678             goto bailout;
679         }
680         b_late_opening = true;
681     }
682
683     if (!b_late_opening && StartMediaCodec(p_dec) != VLC_SUCCESS)
684     {
685         msg_Err(p_dec, "StartMediaCodec failed");
686         goto bailout;
687     }
688
689     if (vlc_clone(&p_sys->out_thread, OutThread, p_dec,
690                   VLC_THREAD_PRIORITY_LOW))
691     {
692         msg_Err(p_dec, "vlc_clone failed");
693         vlc_mutex_unlock(&p_sys->lock);
694         goto bailout;
695     }
696
697     return VLC_SUCCESS;
698
699 bailout:
700     CleanDecoder(p_dec);
701     return VLC_EGENERIC;
702 }
703
704 static int OpenDecoderNdk(vlc_object_t *p_this)
705 {
706     return OpenDecoder(p_this, MediaCodecNdk_Init);
707 }
708
709 static int OpenDecoderJni(vlc_object_t *p_this)
710 {
711     return OpenDecoder(p_this, MediaCodecJni_Init);
712 }
713
714 static void AbortDecoderLocked(decoder_t *p_dec)
715 {
716     decoder_sys_t *p_sys = p_dec->p_sys;
717
718     if (!p_sys->b_aborted)
719     {
720         p_sys->b_aborted = true;
721         vlc_cancel(p_sys->out_thread);
722     }
723 }
724
725 static void CleanDecoder(decoder_t *p_dec)
726 {
727     decoder_sys_t *p_sys = p_dec->p_sys;
728
729     vlc_mutex_destroy(&p_sys->lock);
730     vlc_cond_destroy(&p_sys->cond);
731     vlc_cond_destroy(&p_sys->dec_cond);
732
733     StopMediaCodec(p_dec);
734
735     CSDFree(p_dec);
736     p_sys->api->clean(p_sys->api);
737
738     if (p_dec->fmt_in.i_cat == VIDEO_ES)
739     {
740         if (p_sys->u.video.timestamp_fifo)
741             timestamp_FifoRelease(p_sys->u.video.timestamp_fifo);
742         if (p_sys->u.video.p_awh)
743             AWindowHandler_destroy(p_sys->u.video.p_awh);
744     }
745     free(p_sys->api->psz_name);
746     free(p_sys->api);
747     free(p_sys);
748 }
749
750 /*****************************************************************************
751  * CloseDecoder: Close the decoder instance
752  *****************************************************************************/
753 static void CloseDecoder(vlc_object_t *p_this)
754 {
755     decoder_t *p_dec = (decoder_t *)p_this;
756     decoder_sys_t *p_sys = p_dec->p_sys;
757
758     vlc_mutex_lock(&p_sys->lock);
759     /* Unblock output thread waiting in dequeue_out */
760     DecodeFlushLocked(p_dec);
761     /* Cancel the output thread */
762     AbortDecoderLocked(p_dec);
763     vlc_mutex_unlock(&p_sys->lock);
764
765     vlc_join(p_sys->out_thread, NULL);
766
767     CleanDecoder(p_dec);
768 }
769
770 /*****************************************************************************
771  * vout callbacks
772  *****************************************************************************/
773 static void ReleasePicture(decoder_t *p_dec, unsigned i_index, bool b_render)
774 {
775     decoder_sys_t *p_sys = p_dec->p_sys;
776
777     p_sys->api->release_out(p_sys->api, i_index, b_render);
778 }
779
780 static void InvalidateAllPictures(decoder_t *p_dec)
781 {
782     decoder_sys_t *p_sys = p_dec->p_sys;
783
784     for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
785         AndroidOpaquePicture_Release(p_sys->u.video.pp_inflight_pictures[i],
786                                      false);
787 }
788
789 static int InsertInflightPicture(decoder_t *p_dec, picture_sys_t *p_picsys)
790 {
791     decoder_sys_t *p_sys = p_dec->p_sys;
792
793     if (!p_picsys->priv.hw.p_dec)
794     {
795         p_picsys->priv.hw.p_dec = p_dec;
796         p_picsys->priv.hw.pf_release = ReleasePicture;
797         TAB_APPEND_CAST((picture_sys_t **),
798                         p_sys->u.video.i_inflight_pictures,
799                         p_sys->u.video.pp_inflight_pictures,
800                         p_picsys);
801     } /* else already attached */
802     return 0;
803 }
804
805 static void RemoveInflightPictures(decoder_t *p_dec)
806 {
807     decoder_sys_t *p_sys = p_dec->p_sys;
808
809     for (unsigned int i = 0; i < p_sys->u.video.i_inflight_pictures; ++i)
810         AndroidOpaquePicture_DetachDecoder(p_sys->u.video.pp_inflight_pictures[i]);
811     TAB_CLEAN(p_sys->u.video.i_inflight_pictures,
812               p_sys->u.video.pp_inflight_pictures);
813 }
814
815 static int Video_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
816                                picture_t **pp_out_pic, block_t **pp_out_block)
817 {
818     decoder_sys_t *p_sys = p_dec->p_sys;
819     (void) pp_out_block;
820     assert(pp_out_pic);
821
822     if (p_out->type == MC_OUT_TYPE_BUF)
823     {
824         picture_t *p_pic = NULL;
825
826         /* Use the aspect ratio provided by the input (ie read from packetizer).
827          * Don't check the current value of the aspect ratio in fmt_out, since we
828          * want to allow changes in it to propagate. */
829         if (p_dec->fmt_in.video.i_sar_num != 0 && p_dec->fmt_in.video.i_sar_den != 0
830          && (p_dec->fmt_out.video.i_sar_num != p_dec->fmt_in.video.i_sar_num ||
831              p_dec->fmt_out.video.i_sar_den != p_dec->fmt_in.video.i_sar_den))
832         {
833             p_dec->fmt_out.video.i_sar_num = p_dec->fmt_in.video.i_sar_num;
834             p_dec->fmt_out.video.i_sar_den = p_dec->fmt_in.video.i_sar_den;
835             p_sys->b_update_format = true;
836         }
837
838         if (p_sys->b_update_format)
839         {
840             p_sys->b_update_format = false;
841             if (decoder_UpdateVideoFormat(p_dec) != 0)
842             {
843                 msg_Err(p_dec, "decoder_UpdateVideoFormat failed");
844                 p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
845                 return -1;
846             }
847         }
848
849         /* If the oldest input block had no PTS, the timestamp of
850          * the frame returned by MediaCodec might be wrong so we
851          * overwrite it with the corresponding dts. Call FifoGet
852          * first in order to avoid a gap if buffers are released
853          * due to an invalid format or a preroll */
854         int64_t forced_ts = timestamp_FifoGet(p_sys->u.video.timestamp_fifo);
855
856         if (!p_sys->b_has_format) {
857             msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
858             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
859         }
860
861         if (p_out->u.buf.i_ts <= p_sys->i_preroll_end)
862             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
863
864         if (!p_sys->api->b_direct_rendering && p_out->u.buf.p_ptr == NULL)
865         {
866             /* This can happen when receiving an EOS buffer */
867             msg_Warn(p_dec, "Invalid buffer, dropping frame");
868             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
869         }
870
871         p_pic = decoder_NewPicture(p_dec);
872         if (!p_pic) {
873             msg_Warn(p_dec, "NewPicture failed");
874             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
875         }
876
877         if (forced_ts == VLC_TS_INVALID)
878             p_pic->date = p_out->u.buf.i_ts;
879         else
880             p_pic->date = forced_ts;
881
882         if (p_sys->api->b_direct_rendering)
883         {
884             p_pic->p_sys->priv.hw.i_index = p_out->u.buf.i_index;
885             InsertInflightPicture(p_dec, p_pic->p_sys);
886         } else {
887             unsigned int chroma_div;
888             GetVlcChromaSizes(p_dec->fmt_out.i_codec,
889                               p_dec->fmt_out.video.i_width,
890                               p_dec->fmt_out.video.i_height,
891                               NULL, NULL, &chroma_div);
892             CopyOmxPicture(p_sys->u.video.i_pixel_format, p_pic,
893                            p_sys->u.video.i_slice_height, p_sys->u.video.i_stride,
894                            (uint8_t *)p_out->u.buf.p_ptr, chroma_div, NULL);
895
896             if (p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false))
897             {
898                 picture_Release(p_pic);
899                 return -1;
900             }
901         }
902         assert(!(*pp_out_pic));
903         *pp_out_pic = p_pic;
904         return 1;
905     } else {
906         assert(p_out->type == MC_OUT_TYPE_CONF);
907         p_sys->u.video.i_pixel_format = p_out->u.conf.video.pixel_format;
908
909         const char *name = "unknown";
910         if (p_sys->api->b_direct_rendering)
911             p_dec->fmt_out.i_codec = VLC_CODEC_ANDROID_OPAQUE;
912         else
913         {
914             if (!GetVlcChromaFormat(p_sys->u.video.i_pixel_format,
915                                     &p_dec->fmt_out.i_codec, &name)) {
916                 msg_Err(p_dec, "color-format not recognized");
917                 return -1;
918             }
919         }
920
921         msg_Err(p_dec, "output: %d %s, %dx%d stride %d %d, crop %d %d %d %d",
922                 p_sys->u.video.i_pixel_format, name, p_out->u.conf.video.width, p_out->u.conf.video.height,
923                 p_out->u.conf.video.stride, p_out->u.conf.video.slice_height,
924                 p_out->u.conf.video.crop_left, p_out->u.conf.video.crop_top,
925                 p_out->u.conf.video.crop_right, p_out->u.conf.video.crop_bottom);
926
927         p_dec->fmt_out.video.i_width = p_out->u.conf.video.crop_right + 1 - p_out->u.conf.video.crop_left;
928         p_dec->fmt_out.video.i_height = p_out->u.conf.video.crop_bottom + 1 - p_out->u.conf.video.crop_top;
929         if (p_dec->fmt_out.video.i_width <= 1
930             || p_dec->fmt_out.video.i_height <= 1) {
931             p_dec->fmt_out.video.i_width = p_out->u.conf.video.width;
932             p_dec->fmt_out.video.i_height = p_out->u.conf.video.height;
933         }
934         p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
935         p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
936
937         p_sys->u.video.i_stride = p_out->u.conf.video.stride;
938         p_sys->u.video.i_slice_height = p_out->u.conf.video.slice_height;
939         if (p_sys->u.video.i_stride <= 0)
940             p_sys->u.video.i_stride = p_out->u.conf.video.width;
941         if (p_sys->u.video.i_slice_height <= 0)
942             p_sys->u.video.i_slice_height = p_out->u.conf.video.height;
943
944         if (p_sys->u.video.i_pixel_format == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar)
945             p_sys->u.video.i_slice_height -= p_out->u.conf.video.crop_top/2;
946         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_IGNORE_PADDING))
947         {
948             p_sys->u.video.i_slice_height = 0;
949             p_sys->u.video.i_stride = p_dec->fmt_out.video.i_width;
950         }
951         p_sys->b_update_format = true;
952         p_sys->b_has_format = true;
953         return 0;
954     }
955 }
956
957 /* samples will be in the following order: FL FR FC LFE BL BR BC SL SR */
958 uint32_t pi_audio_order_src[] =
959 {
960     AOUT_CHAN_LEFT, AOUT_CHAN_RIGHT, AOUT_CHAN_CENTER, AOUT_CHAN_LFE,
961     AOUT_CHAN_REARLEFT, AOUT_CHAN_REARRIGHT, AOUT_CHAN_REARCENTER,
962     AOUT_CHAN_MIDDLELEFT, AOUT_CHAN_MIDDLERIGHT,
963 };
964
965 static int Audio_ProcessOutput(decoder_t *p_dec, mc_api_out *p_out,
966                                picture_t **pp_out_pic, block_t **pp_out_block)
967 {
968     decoder_sys_t *p_sys = p_dec->p_sys;
969     (void) pp_out_pic;
970     assert(pp_out_block);
971
972     if (p_out->type == MC_OUT_TYPE_BUF)
973     {
974         block_t *p_block = NULL;
975         if (p_out->u.buf.p_ptr == NULL)
976         {
977             /* This can happen when receiving an EOS buffer */
978             msg_Warn(p_dec, "Invalid buffer, dropping frame");
979             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
980         }
981
982         if (!p_sys->b_has_format) {
983             msg_Warn(p_dec, "Buffers returned before output format is set, dropping frame");
984             return p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false);
985         }
986
987         p_block = block_Alloc(p_out->u.buf.i_size);
988         if (!p_block)
989             return -1;
990         p_block->i_nb_samples = p_out->u.buf.i_size
991                               / p_dec->fmt_out.audio.i_bytes_per_frame;
992
993         if (p_sys->u.audio.b_extract)
994         {
995             aout_ChannelExtract(p_block->p_buffer,
996                                 p_dec->fmt_out.audio.i_channels,
997                                 p_out->u.buf.p_ptr, p_sys->u.audio.i_channels,
998                                 p_block->i_nb_samples, p_sys->u.audio.pi_extraction,
999                                 p_dec->fmt_out.audio.i_bitspersample);
1000         }
1001         else
1002             memcpy(p_block->p_buffer, p_out->u.buf.p_ptr, p_out->u.buf.i_size);
1003
1004         if (p_out->u.buf.i_ts != 0 && p_out->u.buf.i_ts != date_Get(&p_sys->u.audio.i_end_date))
1005             date_Set(&p_sys->u.audio.i_end_date, p_out->u.buf.i_ts);
1006
1007         p_block->i_pts = date_Get(&p_sys->u.audio.i_end_date);
1008         p_block->i_length = date_Increment(&p_sys->u.audio.i_end_date,
1009                                            p_block->i_nb_samples)
1010                           - p_block->i_pts;
1011
1012         if (p_sys->api->release_out(p_sys->api, p_out->u.buf.i_index, false))
1013         {
1014             block_Release(p_block);
1015             return -1;
1016         }
1017         *pp_out_block = p_block;
1018         return 1;
1019     } else {
1020         uint32_t i_layout_dst;
1021         int      i_channels_dst;
1022
1023         assert(p_out->type == MC_OUT_TYPE_CONF);
1024
1025         if (p_out->u.conf.audio.channel_count <= 0
1026          || p_out->u.conf.audio.channel_count > 8
1027          || p_out->u.conf.audio.sample_rate <= 0)
1028         {
1029             msg_Warn( p_dec, "invalid audio properties channels count %d, sample rate %d",
1030                       p_out->u.conf.audio.channel_count,
1031                       p_out->u.conf.audio.sample_rate);
1032             return -1;
1033         }
1034
1035         msg_Err(p_dec, "output: channel_count: %d, channel_mask: 0x%X, rate: %d",
1036                 p_out->u.conf.audio.channel_count, p_out->u.conf.audio.channel_mask,
1037                 p_out->u.conf.audio.sample_rate);
1038
1039         p_dec->fmt_out.i_codec = VLC_CODEC_S16N;
1040         p_dec->fmt_out.audio.i_format = p_dec->fmt_out.i_codec;
1041
1042         p_dec->fmt_out.audio.i_rate = p_out->u.conf.audio.sample_rate;
1043         date_Init(&p_sys->u.audio.i_end_date, p_out->u.conf.audio.sample_rate, 1 );
1044
1045         p_sys->u.audio.i_channels = p_out->u.conf.audio.channel_count;
1046         p_sys->u.audio.b_extract =
1047             aout_CheckChannelExtraction(p_sys->u.audio.pi_extraction,
1048                                         &i_layout_dst, &i_channels_dst,
1049                                         NULL, pi_audio_order_src,
1050                                         p_sys->u.audio.i_channels);
1051
1052         if (p_sys->u.audio.b_extract)
1053             msg_Warn(p_dec, "need channel extraction: %d -> %d",
1054                      p_sys->u.audio.i_channels, i_channels_dst);
1055
1056         p_dec->fmt_out.audio.i_original_channels =
1057         p_dec->fmt_out.audio.i_physical_channels = i_layout_dst;
1058         aout_FormatPrepare(&p_dec->fmt_out.audio);
1059
1060         if (decoder_UpdateAudioFormat(p_dec))
1061             return -1;
1062
1063         p_sys->b_has_format = true;
1064         return 0;
1065     }
1066 }
1067
1068 static void H264ProcessBlock(decoder_t *p_dec, block_t *p_block,
1069                              bool *p_csd_changed, bool *p_size_changed)
1070 {
1071     decoder_sys_t *p_sys = p_dec->p_sys;
1072
1073     assert(p_dec->fmt_in.i_codec == VLC_CODEC_H264 && p_block);
1074
1075     if (p_sys->u.video.i_nal_length_size)
1076     {
1077         h264_AVC_to_AnnexB(p_block->p_buffer, p_block->i_buffer,
1078                                p_sys->u.video.i_nal_length_size);
1079     } else if (H264SetCSD(p_dec, p_block->p_buffer, p_block->i_buffer,
1080                           p_size_changed) == VLC_SUCCESS)
1081     {
1082         *p_csd_changed = true;
1083     }
1084 }
1085
1086 static void HEVCProcessBlock(decoder_t *p_dec, block_t *p_block,
1087                              bool *p_csd_changed, bool *p_size_changed)
1088 {
1089     decoder_sys_t *p_sys = p_dec->p_sys;
1090
1091     assert(p_dec->fmt_in.i_codec == VLC_CODEC_HEVC && p_block);
1092
1093     if (p_sys->u.video.i_nal_length_size)
1094     {
1095         h264_AVC_to_AnnexB(p_block->p_buffer, p_block->i_buffer,
1096                                p_sys->u.video.i_nal_length_size);
1097     }
1098
1099     /* TODO */
1100     VLC_UNUSED(p_csd_changed);
1101     VLC_UNUSED(p_size_changed);
1102 }
1103
1104 static void DecodeFlushLocked(decoder_t *p_dec)
1105 {
1106     decoder_sys_t *p_sys = p_dec->p_sys;
1107     bool b_had_input = p_sys->b_input_dequeued;
1108
1109     p_sys->b_input_dequeued = false;
1110     p_sys->b_flush_out = true;
1111     p_sys->i_preroll_end = 0;
1112     p_sys->b_output_ready = false;
1113     /* Resend CODEC_CONFIG buffer after a flush */
1114     p_sys->i_csd_send = 0;
1115
1116     p_sys->pf_on_flush(p_dec);
1117
1118     if (b_had_input && p_sys->api->flush(p_sys->api) != VLC_SUCCESS)
1119     {
1120         AbortDecoderLocked(p_dec);
1121         return;
1122     }
1123
1124     vlc_cond_broadcast(&p_sys->cond);
1125
1126     while (!p_sys->b_aborted && p_sys->b_flush_out)
1127         vlc_cond_wait(&p_sys->dec_cond, &p_sys->lock);
1128 }
1129
1130 static void DecodeFlush(decoder_t *p_dec)
1131 {
1132     decoder_sys_t *p_sys = p_dec->p_sys;
1133
1134     vlc_mutex_lock(&p_sys->lock);
1135     DecodeFlushLocked(p_dec);
1136     vlc_mutex_unlock(&p_sys->lock);
1137 }
1138
1139 static void *OutThread(void *data)
1140 {
1141     decoder_t *p_dec = data;
1142     decoder_sys_t *p_sys = p_dec->p_sys;
1143
1144     vlc_mutex_lock(&p_sys->lock);
1145     mutex_cleanup_push(&p_sys->lock);
1146     for (;;)
1147     {
1148         int i_index;
1149
1150         /* Wait for output ready */
1151         while (!p_sys->b_flush_out && !p_sys->b_output_ready)
1152             vlc_cond_wait(&p_sys->cond, &p_sys->lock);
1153
1154         if (p_sys->b_flush_out)
1155         {
1156             /* Acknowledge flushed state */
1157             p_sys->b_flush_out = false;
1158             vlc_cond_broadcast(&p_sys->dec_cond);
1159             continue;
1160         }
1161
1162         int canc = vlc_savecancel();
1163
1164         vlc_mutex_unlock(&p_sys->lock);
1165
1166         /* Wait for an output buffer. This function returns when a new output
1167          * is available or if output is flushed. */
1168         i_index = p_sys->api->dequeue_out(p_sys->api, -1);
1169
1170         vlc_mutex_lock(&p_sys->lock);
1171
1172         /* Ignore dequeue_out errors caused by flush */
1173         if (p_sys->b_flush_out)
1174         {
1175             /* If i_index >= 0, Release it. There is no way to know if i_index
1176              * is owned by us, so don't check the error. */
1177             if (i_index >= 0)
1178                 p_sys->api->release_out(p_sys->api, i_index, false);
1179
1180             /* Parse output format/buffers even when we are flushing */
1181             if (i_index != MC_API_INFO_OUTPUT_FORMAT_CHANGED
1182              && i_index != MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1183             {
1184                 vlc_restorecancel(canc);
1185                 continue;
1186             }
1187         }
1188
1189         /* Process output returned by dequeue_out */
1190         if (i_index >= 0 || i_index == MC_API_INFO_OUTPUT_FORMAT_CHANGED
1191          || i_index == MC_API_INFO_OUTPUT_BUFFERS_CHANGED)
1192         {
1193             struct mc_api_out out;
1194             int i_ret = p_sys->api->get_out(p_sys->api, i_index, &out);
1195
1196             if (i_ret == 1)
1197             {
1198                 picture_t *p_pic = NULL;
1199                 block_t *p_block = NULL;
1200
1201                 if (p_sys->pf_process_output(p_dec, &out, &p_pic,
1202                                              &p_block) == -1 && !out.b_eos)
1203                 {
1204                     msg_Err(p_dec, "pf_process_output failed");
1205                     vlc_restorecancel(canc);
1206                     break;
1207                 }
1208                 if (p_pic)
1209                     decoder_QueueVideo(p_dec, p_pic);
1210                 else if (p_block)
1211                     decoder_QueueAudio(p_dec, p_block);
1212
1213                 if (out.b_eos)
1214                 {
1215                     msg_Warn(p_dec, "EOS received, abort OutThread");
1216                     vlc_restorecancel(canc);
1217                     break;
1218                 }
1219             } else if (i_ret != 0)
1220             {
1221                 msg_Err(p_dec, "get_out failed");
1222                 vlc_restorecancel(canc);
1223                 break;
1224             }
1225         }
1226         else
1227         {
1228             msg_Err(p_dec, "dequeue_out failed");
1229             vlc_restorecancel(canc);
1230             break;
1231         }
1232         vlc_restorecancel(canc);
1233     }
1234     msg_Warn(p_dec, "OutThread stopped");
1235
1236     /* Signal DecoderFlush that the output thread aborted */
1237     p_sys->b_aborted = true;
1238     vlc_cond_signal(&p_sys->dec_cond);
1239
1240     vlc_cleanup_pop();
1241     vlc_mutex_unlock(&p_sys->lock);
1242
1243     return NULL;
1244 }
1245
1246 static block_t *GetNextBlock(decoder_sys_t *p_sys, block_t *p_block)
1247 {
1248     if (p_sys->i_csd_send < p_sys->i_csd_count)
1249         return p_sys->pp_csd[p_sys->i_csd_send++];
1250     else
1251         return p_block;
1252 }
1253
1254 /**
1255  * DecodeCommon called from DecodeVideo or DecodeAudio.
1256  * It returns -1 in case of error, 0 otherwise.
1257  */
1258 static int DecodeCommon(decoder_t *p_dec, block_t **pp_block)
1259 {
1260     decoder_sys_t *p_sys = p_dec->p_sys;
1261     int i_flags = 0;
1262     int i_ret;
1263     bool b_dequeue_timeout = false;
1264     bool b_draining;
1265
1266     if (pp_block != NULL && *pp_block == NULL)
1267         return 0;
1268
1269     vlc_mutex_lock(&p_sys->lock);
1270
1271     if (p_sys->b_aborted)
1272         goto end;
1273
1274     if (pp_block != NULL)
1275     {
1276         block_t *p_block = *pp_block;
1277         b_draining = false;
1278         if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED))
1279         {
1280             DecodeFlushLocked(p_dec);
1281             if (p_sys->b_aborted)
1282                 goto end;
1283             if (p_block->i_flags & BLOCK_FLAG_CORRUPTED)
1284                 goto end;
1285         }
1286
1287         /* Parse input block */
1288         if ((i_ret = p_sys->pf_on_new_block(p_dec, p_block, &i_flags)) == 1)
1289         {
1290             if (i_flags & (NEWBLOCK_FLAG_FLUSH|NEWBLOCK_FLAG_RESTART))
1291             {
1292                 msg_Warn(p_dec, "Flushing from DecodeCommon");
1293
1294                 /* Flush before restart to unblock OutThread */
1295                 DecodeFlushLocked(p_dec);
1296                 if (p_sys->b_aborted)
1297                     goto end;
1298
1299                 if (i_flags & NEWBLOCK_FLAG_RESTART)
1300                 {
1301                     msg_Warn(p_dec, "Restarting from DecodeCommon");
1302                     StopMediaCodec(p_dec);
1303                     if (StartMediaCodec(p_dec) != VLC_SUCCESS)
1304                     {
1305                         msg_Err(p_dec, "StartMediaCodec failed");
1306                         AbortDecoderLocked(p_dec);
1307                         goto end;
1308                     }
1309                 }
1310             }
1311         }
1312         else
1313         {
1314             if (i_ret != 0)
1315             {
1316                 AbortDecoderLocked(p_dec);
1317                 msg_Err(p_dec, "pf_on_new_block failed");
1318             }
1319             goto end;
1320         }
1321     }
1322     else
1323     {
1324         /* No input block, decoder is draining */
1325         msg_Err(p_dec, "Decoder is draining");
1326         b_draining = true;
1327
1328         if (!p_sys->b_output_ready )
1329         {
1330             /* Output no ready, no need to drain */
1331             goto end;
1332         }
1333     }
1334
1335     /* Abort if MediaCodec is not yet started */
1336     if (!p_sys->api->b_started)
1337         goto end;
1338
1339     /* Queue CSD blocks and input blocks */
1340     block_t *p_block = NULL;
1341     while (b_draining || (p_block = GetNextBlock(p_sys, *pp_block)))
1342     {
1343         int i_index;
1344
1345         vlc_mutex_unlock(&p_sys->lock);
1346         /* Wait for an input buffer. This function returns when a new input
1347          * buffer is available or after 1sec of timeout. */
1348         i_index = p_sys->api->dequeue_in(p_sys->api,
1349                                          p_sys->api->b_direct_rendering ?
1350                                          INT64_C(1000000) : -1);
1351         vlc_mutex_lock(&p_sys->lock);
1352
1353         if (p_sys->b_aborted)
1354             goto end;
1355
1356         bool b_config = false;
1357         mtime_t i_ts = 0;
1358         p_sys->b_input_dequeued = true;
1359         const void *p_buf = NULL;
1360         size_t i_size = 0;
1361
1362         if (i_index >= 0)
1363         {
1364             assert(b_draining || p_block != NULL);
1365             if (p_block != NULL)
1366             {
1367                 b_config = (p_block->i_flags & BLOCK_FLAG_CSD);
1368                 if (!b_config)
1369                 {
1370                     i_ts = p_block->i_pts;
1371                     if (!i_ts && p_block->i_dts)
1372                         i_ts = p_block->i_dts;
1373                 }
1374                 p_buf = p_block->p_buffer;
1375                 i_size = p_block->i_buffer;
1376             }
1377
1378             if (p_sys->api->queue_in(p_sys->api, i_index, p_buf, i_size,
1379                                      i_ts, b_config) == 0)
1380             {
1381                 if (!b_config && p_block != NULL)
1382                 {
1383                     if (p_block->i_flags & BLOCK_FLAG_PREROLL )
1384                         p_sys->i_preroll_end = i_ts;
1385
1386                     /* One input buffer is queued, signal OutThread that will
1387                      * fetch output buffers */
1388                     p_sys->b_output_ready = true;
1389                     vlc_cond_broadcast(&p_sys->cond);
1390
1391                     assert(p_block == *pp_block),
1392                     block_Release(p_block);
1393                     *pp_block = NULL;
1394                 }
1395                 b_dequeue_timeout = false;
1396                 if (b_draining)
1397                     break;
1398             } else
1399             {
1400                 msg_Err(p_dec, "queue_in failed");
1401                 AbortDecoderLocked(p_dec);
1402                 goto end;
1403             }
1404         }
1405         else if (i_index == MC_API_INFO_TRYAGAIN)
1406         {
1407             /* HACK: When direct rendering is enabled, there is a possible
1408              * deadlock between the Decoder and the Vout. It happens when the
1409              * Vout is paused and when the Decoder is flushing. In that case,
1410              * the Vout won't release any output buffers, therefore MediaCodec
1411              * won't dequeue any input buffers. To work around this issue,
1412              * release all output buffers if DecodeCommon is waiting more than
1413              * 1sec for a new input buffer. */
1414             if (!b_dequeue_timeout)
1415             {
1416                 msg_Warn(p_dec, "Decoder stuck: invalidate all buffers");
1417                 InvalidateAllPictures(p_dec);
1418                 b_dequeue_timeout = true;
1419                 continue;
1420             }
1421             else
1422             {
1423                 msg_Err(p_dec, "dequeue_in timeout: no input available for 2secs");
1424                 AbortDecoderLocked(p_dec);
1425                 goto end;
1426             }
1427         }
1428         else
1429         {
1430             msg_Err(p_dec, "dequeue_in failed");
1431             AbortDecoderLocked(p_dec);
1432             goto end;
1433         }
1434     }
1435
1436     if (b_draining)
1437     {
1438         msg_Warn(p_dec, "EOS sent, waiting for OutThread");
1439
1440         /* Wait for the OutThread to stop (and process all remaining output
1441          * frames. Use a timeout here since we can't know if all decoders will
1442          * behave correctly. */
1443         mtime_t deadline = mdate() + INT64_C(1000000);
1444         while (!p_sys->b_aborted
1445             && vlc_cond_timedwait(&p_sys->dec_cond, &p_sys->lock, deadline) == 0);
1446
1447         if (!p_sys->b_aborted)
1448             msg_Err(p_dec, "OutThread timed out");
1449
1450         /* In case pf_decode is called again (it shouldn't happen) */
1451         p_sys->b_error_signaled = true;
1452
1453         vlc_mutex_unlock(&p_sys->lock);
1454         return 0;
1455     }
1456
1457 end:
1458     if (pp_block && *pp_block)
1459     {
1460         block_Release(*pp_block);
1461         *pp_block = NULL;
1462     }
1463     if (p_sys->b_aborted)
1464     {
1465         if (!p_sys->b_error_signaled) {
1466             /* Signal the error to the Java.
1467              * TODO: remove this when there is a decoder fallback */
1468             if (p_dec->fmt_in.i_cat == VIDEO_ES)
1469                 AWindowHandler_sendHardwareAccelerationError(VLC_OBJECT(p_dec),
1470                                                              p_sys->u.video.p_awh);
1471             p_sys->b_error_signaled = true;
1472             vlc_cond_broadcast(&p_sys->cond);
1473         }
1474         vlc_mutex_unlock(&p_sys->lock);
1475         return -1;
1476     }
1477     else
1478     {
1479         vlc_mutex_unlock(&p_sys->lock);
1480         return 0;
1481     }
1482 }
1483
1484 static int Video_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
1485 {
1486     decoder_sys_t *p_sys = p_dec->p_sys;
1487     bool b_csd_changed = false, b_size_changed = false;
1488
1489     if (p_block->i_flags & BLOCK_FLAG_INTERLACED_MASK
1490         && !p_sys->api->b_support_interlaced)
1491         return -1;
1492
1493     if (p_dec->fmt_in.i_codec == VLC_CODEC_H264)
1494         H264ProcessBlock(p_dec, p_block, &b_csd_changed, &b_size_changed);
1495     else if (p_dec->fmt_in.i_codec == VLC_CODEC_HEVC)
1496         HEVCProcessBlock(p_dec, p_block, &b_csd_changed, &b_size_changed);
1497
1498     if (b_csd_changed)
1499     {
1500         if (b_size_changed || !p_sys->api->b_started)
1501         {
1502             if (p_sys->api->b_started)
1503                 msg_Err(p_dec, "SPS/PPS changed during playback and "
1504                         "video size are different. Restart it !");
1505             *p_flags |= NEWBLOCK_FLAG_RESTART;
1506         } else
1507         {
1508             msg_Err(p_dec, "SPS/PPS changed during playback. Flush it");
1509             *p_flags |= NEWBLOCK_FLAG_FLUSH;
1510         }
1511     }
1512
1513     if (!p_sys->api->b_started)
1514     {
1515         *p_flags |= NEWBLOCK_FLAG_RESTART;
1516
1517         /* Don't start if we don't have any csd */
1518         if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
1519          && !p_dec->fmt_in.i_extra && !p_sys->pp_csd)
1520             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1521
1522         /* Don't start if we don't have a valid video size */
1523         if ((p_sys->i_quirks & OMXCODEC_VIDEO_QUIRKS_NEED_SIZE)
1524          && (!p_sys->u.video.i_width || !p_sys->u.video.i_height))
1525             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1526     }
1527
1528     timestamp_FifoPut(p_sys->u.video.timestamp_fifo,
1529                       p_block->i_pts ? VLC_TS_INVALID : p_block->i_dts);
1530
1531     return 1;
1532 }
1533
1534 static void Video_OnFlush(decoder_t *p_dec)
1535 {
1536     decoder_sys_t *p_sys = p_dec->p_sys;
1537
1538     timestamp_FifoEmpty(p_sys->u.video.timestamp_fifo);
1539     /* Invalidate all pictures that are currently in flight
1540      * since flushing make all previous indices returned by
1541      * MediaCodec invalid. */
1542     if (p_sys->api->b_direct_rendering)
1543         InvalidateAllPictures(p_dec);
1544 }
1545
1546 static picture_t *DecodeVideo(decoder_t *p_dec, block_t **pp_block)
1547 {
1548     DecodeCommon(p_dec, pp_block);
1549     return NULL;
1550 }
1551
1552 static int Audio_OnNewBlock(decoder_t *p_dec, block_t *p_block, int *p_flags)
1553 {
1554     decoder_sys_t *p_sys = p_dec->p_sys;
1555
1556     /* We've just started the stream, wait for the first PTS. */
1557     if (!date_Get(&p_sys->u.audio.i_end_date))
1558     {
1559         if (p_block->i_pts <= VLC_TS_INVALID)
1560             return 0;
1561         date_Set(&p_sys->u.audio.i_end_date, p_block->i_pts);
1562     }
1563
1564     /* try delayed opening if there is a new extra data */
1565     if (!p_sys->api->b_started)
1566     {
1567         p_dec->p_sys->u.audio.i_channels = p_dec->fmt_in.audio.i_channels;
1568
1569         *p_flags |= NEWBLOCK_FLAG_RESTART;
1570
1571         /* Don't start if we don't have any csd */
1572         if ((p_sys->i_quirks & OMXCODEC_QUIRKS_NEED_CSD)
1573          && !p_dec->fmt_in.i_extra)
1574             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1575
1576         /* Don't start if we don't have a valid channels count */
1577         if ((p_sys->i_quirks & OMXCODEC_AUDIO_QUIRKS_NEED_CHANNELS)
1578          && !p_dec->p_sys->u.audio.i_channels)
1579             *p_flags &= ~NEWBLOCK_FLAG_RESTART;
1580     }
1581     return 1;
1582 }
1583
1584 static void Audio_OnFlush(decoder_t *p_dec)
1585 {
1586     decoder_sys_t *p_sys = p_dec->p_sys;
1587
1588     date_Set(&p_sys->u.audio.i_end_date, VLC_TS_INVALID);
1589 }
1590
1591 static block_t *DecodeAudio(decoder_t *p_dec, block_t **pp_block)
1592 {
1593     DecodeCommon(p_dec, pp_block);
1594     return NULL;
1595 }