Merge remote-tracking branch 'qatar/master'
[ffmpeg.git] / libavcodec / libstagefright.cpp
1 /*
2  * Interface to the Android Stagefright library for
3  * H/W accelerated H.264 decoding
4  *
5  * Copyright (C) 2011 Mohamed Naufal
6  * Copyright (C) 2011 Martin Storsj√∂
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 #include <binder/ProcessState.h>
26 #include <media/stagefright/MetaData.h>
27 #include <media/stagefright/MediaBufferGroup.h>
28 #include <media/stagefright/MediaDebug.h>
29 #include <media/stagefright/MediaDefs.h>
30 #include <media/stagefright/OMXClient.h>
31 #include <media/stagefright/OMXCodec.h>
32 #include <utils/List.h>
33 #include <new>
34
35 extern "C" {
36 #include "avcodec.h"
37 #include "libavutil/imgutils.h"
38 }
39
40 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
41
42 using namespace android;
43
44 struct Frame {
45     status_t status;
46     size_t size;
47     int64_t time;
48     int key;
49     uint8_t *buffer;
50     MediaBuffer* mbuffer;
51     int32_t w, h;
52 };
53
54 class CustomSource;
55
56 struct StagefrightContext {
57     AVCodecContext *avctx;
58     AVBitStreamFilterContext *bsfc;
59     uint8_t* orig_extradata;
60     int orig_extradata_size;
61     sp<MediaSource> *source;
62     List<Frame*> *in_queue, *out_queue;
63     pthread_mutex_t in_mutex, out_mutex;
64     pthread_cond_t condition;
65     pthread_t decode_thread_id;
66
67     Frame *end_frame;
68     bool source_done;
69     volatile sig_atomic_t thread_started, thread_exited, stop_decode;
70
71     AVFrame ret_frame;
72
73     uint8_t *dummy_buf;
74     int dummy_bufsize;
75
76     OMXClient *client;
77     sp<MediaSource> *decoder;
78     const char *decoder_component;
79 };
80
81 class CustomSource : public MediaSource {
82 public:
83     CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
84         s = (StagefrightContext*)avctx->priv_data;
85         source_meta = meta;
86         frame_size  = (avctx->width * avctx->height * 3) / 2;
87         buf_group.add_buffer(new MediaBuffer(frame_size));
88     }
89
90     virtual sp<MetaData> getFormat() {
91         return source_meta;
92     }
93
94     virtual status_t start(MetaData *params) {
95         return OK;
96     }
97
98     virtual status_t stop() {
99         return OK;
100     }
101
102     virtual status_t read(MediaBuffer **buffer,
103                           const MediaSource::ReadOptions *options) {
104         Frame *frame;
105         status_t ret;
106
107         if (s->thread_exited)
108             return ERROR_END_OF_STREAM;
109         pthread_mutex_lock(&s->in_mutex);
110
111         while (s->in_queue->empty())
112             pthread_cond_wait(&s->condition, &s->in_mutex);
113
114         frame = *s->in_queue->begin();
115         ret = frame->status;
116
117         if (ret == OK) {
118             ret = buf_group.acquire_buffer(buffer);
119             if (ret == OK) {
120                 memcpy((*buffer)->data(), frame->buffer, frame->size);
121                 (*buffer)->set_range(0, frame->size);
122                 (*buffer)->meta_data()->clear();
123                 (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
124                 (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
125             } else {
126                 av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
127             }
128             av_freep(&frame->buffer);
129         }
130
131         s->in_queue->erase(s->in_queue->begin());
132         pthread_mutex_unlock(&s->in_mutex);
133
134         av_freep(&frame);
135         return ret;
136     }
137
138 private:
139     MediaBufferGroup buf_group;
140     sp<MetaData> source_meta;
141     StagefrightContext *s;
142     int frame_size;
143 };
144
145 void* decode_thread(void *arg)
146 {
147     AVCodecContext *avctx = (AVCodecContext*)arg;
148     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
149     Frame* frame;
150     MediaBuffer *buffer;
151     int decode_done = 0;
152     do {
153         buffer = NULL;
154         frame = (Frame*)av_mallocz(sizeof(Frame));
155         if (!frame) {
156             frame         = s->end_frame;
157             frame->status = AVERROR(ENOMEM);
158             decode_done   = 1;
159             s->end_frame  = NULL;
160         } else {
161             frame->status = (*s->decoder)->read(&buffer);
162             if (frame->status == OK) {
163                 sp<MetaData> outFormat = (*s->decoder)->getFormat();
164                 outFormat->findInt32(kKeyWidth , &frame->w);
165                 outFormat->findInt32(kKeyHeight, &frame->h);
166                 frame->size    = buffer->range_length();
167                 frame->mbuffer = buffer;
168             } else if (frame->status == INFO_FORMAT_CHANGED) {
169                 if (buffer)
170                     buffer->release();
171                 av_free(frame);
172                 continue;
173             } else {
174                 decode_done = 1;
175             }
176         }
177         while (true) {
178             pthread_mutex_lock(&s->out_mutex);
179             if (s->out_queue->size() >= 10) {
180                 pthread_mutex_unlock(&s->out_mutex);
181                 usleep(10000);
182                 continue;
183             }
184             break;
185         }
186         s->out_queue->push_back(frame);
187         pthread_mutex_unlock(&s->out_mutex);
188     } while (!decode_done && !s->stop_decode);
189
190     s->thread_exited = true;
191
192     return 0;
193 }
194
195 static av_cold int Stagefright_init(AVCodecContext *avctx)
196 {
197     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
198     sp<MetaData> meta, outFormat;
199     int32_t colorFormat = 0;
200     int ret;
201
202     if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
203         return -1;
204
205     s->avctx = avctx;
206     s->bsfc  = av_bitstream_filter_init("h264_mp4toannexb");
207     if (!s->bsfc) {
208         av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
209         return -1;
210     }
211
212     s->orig_extradata_size = avctx->extradata_size;
213     s->orig_extradata = (uint8_t*) av_mallocz(avctx->extradata_size +
214                                               FF_INPUT_BUFFER_PADDING_SIZE);
215     if (!s->orig_extradata) {
216         ret = AVERROR(ENOMEM);
217         goto fail;
218     }
219     memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
220
221     meta = new MetaData;
222     if (meta == NULL) {
223         ret = AVERROR(ENOMEM);
224         goto fail;
225     }
226     meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
227     meta->setInt32(kKeyWidth, avctx->width);
228     meta->setInt32(kKeyHeight, avctx->height);
229     meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
230
231     android::ProcessState::self()->startThreadPool();
232
233     s->source    = new sp<MediaSource>();
234     *s->source   = new CustomSource(avctx, meta);
235     s->in_queue  = new List<Frame*>;
236     s->out_queue = new List<Frame*>;
237     s->client    = new OMXClient;
238     s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
239     if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
240         !s->end_frame) {
241         ret = AVERROR(ENOMEM);
242         goto fail;
243     }
244
245     if (s->client->connect() !=  OK) {
246         av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
247         ret = -1;
248         goto fail;
249     }
250
251     s->decoder  = new sp<MediaSource>();
252     *s->decoder = OMXCodec::Create(s->client->interface(), meta,
253                                   false, *s->source, NULL,
254                                   OMXCodec::kClientNeedsFramebuffer);
255     if ((*s->decoder)->start() !=  OK) {
256         av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
257         ret = -1;
258         s->client->disconnect();
259         goto fail;
260     }
261
262     outFormat = (*s->decoder)->getFormat();
263     outFormat->findInt32(kKeyColorFormat, &colorFormat);
264     if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
265         colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
266         avctx->pix_fmt = PIX_FMT_NV21;
267     else
268         avctx->pix_fmt = PIX_FMT_YUV420P;
269
270     outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
271     if (s->decoder_component)
272         s->decoder_component = av_strdup(s->decoder_component);
273
274     pthread_mutex_init(&s->in_mutex, NULL);
275     pthread_mutex_init(&s->out_mutex, NULL);
276     pthread_cond_init(&s->condition, NULL);
277     return 0;
278
279 fail:
280     av_bitstream_filter_close(s->bsfc);
281     av_freep(&s->orig_extradata);
282     av_freep(&s->end_frame);
283     delete s->in_queue;
284     delete s->out_queue;
285     delete s->client;
286     return ret;
287 }
288
289 static int Stagefright_decode_frame(AVCodecContext *avctx, void *data,
290                                     int *data_size, AVPacket *avpkt)
291 {
292     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
293     Frame *frame;
294     MediaBuffer *mbuffer;
295     status_t status;
296     size_t size;
297     uint8_t *buf;
298     const uint8_t *src_data[3];
299     int w, h;
300     int src_linesize[3];
301     int orig_size = avpkt->size;
302     AVPacket pkt = *avpkt;
303     int ret;
304
305     if (!s->thread_started) {
306         pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx);
307         s->thread_started = true;
308     }
309
310     if (avpkt && avpkt->data) {
311         av_bitstream_filter_filter(s->bsfc, avctx, NULL, &pkt.data, &pkt.size,
312                                    avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
313         avpkt = &pkt;
314     }
315
316     if (!s->source_done) {
317         if(!s->dummy_buf) {
318             s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
319             if (!s->dummy_buf)
320                 return AVERROR(ENOMEM);
321             s->dummy_bufsize = avpkt->size;
322             memcpy(s->dummy_buf, avpkt->data, avpkt->size);
323         }
324
325         frame = (Frame*)av_mallocz(sizeof(Frame));
326         if (avpkt->data) {
327             frame->status  = OK;
328             frame->size    = avpkt->size;
329             // Stagefright can't handle negative timestamps -
330             // if needed, work around this by offsetting them manually?
331             if (avpkt->pts >= 0)
332                 frame->time    = avpkt->pts;
333             frame->key     = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
334             frame->buffer  = (uint8_t*)av_malloc(avpkt->size);
335             if (!frame->buffer) {
336                 av_freep(&frame);
337                 return AVERROR(ENOMEM);
338             }
339             uint8_t *ptr = avpkt->data;
340             // The OMX.SEC decoder fails without this.
341             if (avpkt->size == orig_size + avctx->extradata_size) {
342                 ptr += avctx->extradata_size;
343                 frame->size = orig_size;
344             }
345             memcpy(frame->buffer, ptr, orig_size);
346         } else {
347             frame->status  = ERROR_END_OF_STREAM;
348             s->source_done = true;
349         }
350
351         while (true) {
352             if (s->thread_exited) {
353                 s->source_done = true;
354                 break;
355             }
356             pthread_mutex_lock(&s->in_mutex);
357             if (s->in_queue->size() >= 10) {
358                 pthread_mutex_unlock(&s->in_mutex);
359                 usleep(10000);
360                 continue;
361             }
362             s->in_queue->push_back(frame);
363             pthread_cond_signal(&s->condition);
364             pthread_mutex_unlock(&s->in_mutex);
365             break;
366         }
367     }
368     while (true) {
369         pthread_mutex_lock(&s->out_mutex);
370         if (!s->out_queue->empty()) break;
371         pthread_mutex_unlock(&s->out_mutex);
372         if (s->source_done) {
373             usleep(10000);
374             continue;
375         } else {
376             return orig_size;
377         }
378     }
379
380     frame = *s->out_queue->begin();
381     s->out_queue->erase(s->out_queue->begin());
382     pthread_mutex_unlock(&s->out_mutex);
383
384     mbuffer = frame->mbuffer;
385     status  = frame->status;
386     size    = frame->size;
387     w       = frame->w;
388     h       = frame->h;
389     av_freep(&frame);
390
391     if (status == ERROR_END_OF_STREAM)
392         return 0;
393     if (status != OK) {
394         if (status == AVERROR(ENOMEM))
395             return status;
396         av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
397         return -1;
398     }
399
400     // The OMX.SEC decoder doesn't signal the modified width/height
401     if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
402         (w & 15 || h & 15)) {
403         if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == size) {
404             w = (w + 15)&~15;
405             h = (h + 15)&~15;
406         }
407     }
408
409     if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
410         avctx->width  = w;
411         avctx->height = h;
412     }
413
414     ret = avctx->reget_buffer(avctx, &s->ret_frame);
415     if (ret < 0) {
416         av_log(avctx, AV_LOG_ERROR, "reget buffer() failed\n");
417         goto end;
418     }
419
420     src_linesize[0] = w;
421     if (avctx->pix_fmt == PIX_FMT_YUV420P)
422         src_linesize[1] = src_linesize[2] = w/2;
423     else if (avctx->pix_fmt == PIX_FMT_NV21)
424         src_linesize[1] = w;
425
426     buf = (uint8_t*)mbuffer->data();
427     src_data[0] = buf;
428     src_data[1] = buf + src_linesize[0] * h;
429     src_data[2] = src_data[1] + src_linesize[1] * h/2;
430     av_image_copy(s->ret_frame.data, s->ret_frame.linesize,
431                   src_data, src_linesize,
432                   avctx->pix_fmt, avctx->width, avctx->height);
433
434     *data_size = sizeof(AVFrame);
435     *(AVFrame*)data = s->ret_frame;
436     ret = orig_size;
437 end:
438     mbuffer->release();
439     return ret;
440 }
441
442 static av_cold int Stagefright_close(AVCodecContext *avctx)
443 {
444     StagefrightContext *s = (StagefrightContext*)avctx->priv_data;
445     Frame *frame;
446
447     if (s->thread_started) {
448         if (!s->thread_exited) {
449             s->stop_decode = 1;
450
451             // Make sure decode_thread() doesn't get stuck
452             pthread_mutex_lock(&s->out_mutex);
453             while (!s->out_queue->empty()) {
454                 frame = *s->out_queue->begin();
455                 s->out_queue->erase(s->out_queue->begin());
456                 if (frame->size)
457                     frame->mbuffer->release();
458                 av_freep(&frame);
459             }
460             pthread_mutex_unlock(&s->out_mutex);
461
462             // Feed a dummy frame prior to signalling EOF.
463             // This is required to terminate the decoder(OMX.SEC)
464             // when only one frame is read during stream info detection.
465             if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
466                 frame->status = OK;
467                 frame->size   = s->dummy_bufsize;
468                 frame->key    = 1;
469                 frame->buffer = s->dummy_buf;
470                 pthread_mutex_lock(&s->in_mutex);
471                 s->in_queue->push_back(frame);
472                 pthread_cond_signal(&s->condition);
473                 pthread_mutex_unlock(&s->in_mutex);
474                 s->dummy_buf = NULL;
475             }
476
477             pthread_mutex_lock(&s->in_mutex);
478             s->end_frame->status = ERROR_END_OF_STREAM;
479             s->in_queue->push_back(s->end_frame);
480             pthread_cond_signal(&s->condition);
481             pthread_mutex_unlock(&s->in_mutex);
482             s->end_frame = NULL;
483         }
484
485         pthread_join(s->decode_thread_id, NULL);
486
487         if (s->ret_frame.data[0])
488             avctx->release_buffer(avctx, &s->ret_frame);
489
490         s->thread_started = false;
491     }
492
493     while (!s->in_queue->empty()) {
494         frame = *s->in_queue->begin();
495         s->in_queue->erase(s->in_queue->begin());
496         if (frame->size)
497             av_freep(&frame->buffer);
498         av_freep(&frame);
499     }
500
501     while (!s->out_queue->empty()) {
502         frame = *s->out_queue->begin();
503         s->out_queue->erase(s->out_queue->begin());
504         if (frame->size)
505             frame->mbuffer->release();
506         av_freep(&frame);
507     }
508
509     (*s->decoder)->stop();
510     s->client->disconnect();
511
512     if (s->decoder_component)
513         av_freep(&s->decoder_component);
514     av_freep(&s->dummy_buf);
515     av_freep(&s->end_frame);
516
517     // Reset the extradata back to the original mp4 format, so that
518     // the next invocation (both when decoding and when called from
519     // av_find_stream_info) get the original mp4 format extradata.
520     av_freep(&avctx->extradata);
521     avctx->extradata = s->orig_extradata;
522     avctx->extradata_size = s->orig_extradata_size;
523
524     delete s->in_queue;
525     delete s->out_queue;
526     delete s->client;
527     delete s->decoder;
528     delete s->source;
529
530     pthread_mutex_destroy(&s->in_mutex);
531     pthread_mutex_destroy(&s->out_mutex);
532     pthread_cond_destroy(&s->condition);
533     av_bitstream_filter_close(s->bsfc);
534     return 0;
535 }
536
537 AVCodec ff_libstagefright_h264_decoder = {
538     "libstagefright_h264",
539     AVMEDIA_TYPE_VIDEO,
540     CODEC_ID_H264,
541     sizeof(StagefrightContext),
542     Stagefright_init,
543     NULL, //encode
544     Stagefright_close,
545     Stagefright_decode_frame,
546     CODEC_CAP_DELAY,
547     NULL, //next
548     NULL, //flush
549     NULL, //supported_framerates
550     NULL, //pixel_formats
551     NULL_IF_CONFIG_SMALL("libstagefright H.264"),
552 };