2 * RockChip MPP Video Decoder
3 * Copyright (c) 2017 Lionel CHAZALLON
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 #include <drm_fourcc.h>
24 #include <rockchip/mpp_buffer.h>
25 #include <rockchip/rk_mpi.h>
33 #include "libavutil/buffer.h"
34 #include "libavutil/common.h"
35 #include "libavutil/frame.h"
36 #include "libavutil/hwcontext.h"
37 #include "libavutil/hwcontext_drm.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/log.h"
41 #define RECEIVE_FRAME_TIMEOUT 100
42 #define FRAMEGROUP_MAX_FRAMES 16
43 #define INPUT_MAX_PACKETS 4
48 MppBufferGroup frame_group;
54 AVBufferRef *frames_ref;
55 AVBufferRef *device_ref;
60 AVBufferRef *decoder_ref;
65 AVBufferRef *decoder_ref;
68 static MppCodingType rkmpp_get_codingtype(AVCodecContext *avctx)
70 switch (avctx->codec_id) {
71 case AV_CODEC_ID_H264: return MPP_VIDEO_CodingAVC;
72 case AV_CODEC_ID_HEVC: return MPP_VIDEO_CodingHEVC;
73 case AV_CODEC_ID_VP8: return MPP_VIDEO_CodingVP8;
74 case AV_CODEC_ID_VP9: return MPP_VIDEO_CodingVP9;
75 default: return MPP_VIDEO_CodingUnused;
79 static uint32_t rkmpp_get_frameformat(MppFrameFormat mppformat)
82 case MPP_FMT_YUV420SP: return DRM_FORMAT_NV12;
83 #ifdef DRM_FORMAT_NV12_10
84 case MPP_FMT_YUV420SP_10BIT: return DRM_FORMAT_NV12_10;
90 static int rkmpp_write_data(AVCodecContext *avctx, uint8_t *buffer, int size, int64_t pts)
92 RKMPPDecodeContext *rk_context = avctx->priv_data;
93 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
97 // create the MPP packet
98 ret = mpp_packet_init(&packet, buffer, size);
100 av_log(avctx, AV_LOG_ERROR, "Failed to init MPP packet (code = %d)\n", ret);
101 return AVERROR_UNKNOWN;
104 mpp_packet_set_pts(packet, pts);
107 mpp_packet_set_eos(packet);
109 ret = decoder->mpi->decode_put_packet(decoder->ctx, packet);
111 if (ret == MPP_ERR_BUFFER_FULL) {
112 av_log(avctx, AV_LOG_DEBUG, "Buffer full writing %d bytes to decoder\n", size);
113 ret = AVERROR(EAGAIN);
115 ret = AVERROR_UNKNOWN;
118 av_log(avctx, AV_LOG_DEBUG, "Wrote %d bytes to decoder\n", size);
120 mpp_packet_deinit(&packet);
125 static int rkmpp_close_decoder(AVCodecContext *avctx)
127 RKMPPDecodeContext *rk_context = avctx->priv_data;
128 av_buffer_unref(&rk_context->decoder_ref);
132 static void rkmpp_release_decoder(void *opaque, uint8_t *data)
134 RKMPPDecoder *decoder = (RKMPPDecoder *)data;
137 decoder->mpi->reset(decoder->ctx);
138 mpp_destroy(decoder->ctx);
142 if (decoder->frame_group) {
143 mpp_buffer_group_put(decoder->frame_group);
144 decoder->frame_group = NULL;
147 av_buffer_unref(&decoder->frames_ref);
148 av_buffer_unref(&decoder->device_ref);
153 static int rkmpp_init_decoder(AVCodecContext *avctx)
155 RKMPPDecodeContext *rk_context = avctx->priv_data;
156 RKMPPDecoder *decoder = NULL;
157 MppCodingType codectype = MPP_VIDEO_CodingUnused;
162 avctx->pix_fmt = AV_PIX_FMT_DRM_PRIME;
164 // create a decoder and a ref to it
165 decoder = av_mallocz(sizeof(RKMPPDecoder));
167 ret = AVERROR(ENOMEM);
171 rk_context->decoder_ref = av_buffer_create((uint8_t *)decoder, sizeof(*decoder), rkmpp_release_decoder,
172 NULL, AV_BUFFER_FLAG_READONLY);
173 if (!rk_context->decoder_ref) {
175 ret = AVERROR(ENOMEM);
179 av_log(avctx, AV_LOG_DEBUG, "Initializing RKMPP decoder.\n");
181 codectype = rkmpp_get_codingtype(avctx);
182 if (codectype == MPP_VIDEO_CodingUnused) {
183 av_log(avctx, AV_LOG_ERROR, "Unknown codec type (%d).\n", avctx->codec_id);
184 ret = AVERROR_UNKNOWN;
188 ret = mpp_check_support_format(MPP_CTX_DEC, codectype);
190 av_log(avctx, AV_LOG_ERROR, "Codec type (%d) unsupported by MPP\n", avctx->codec_id);
191 ret = AVERROR_UNKNOWN;
195 // Create the MPP context
196 ret = mpp_create(&decoder->ctx, &decoder->mpi);
198 av_log(avctx, AV_LOG_ERROR, "Failed to create MPP context (code = %d).\n", ret);
199 ret = AVERROR_UNKNOWN;
204 ret = mpp_init(decoder->ctx, MPP_CTX_DEC, codectype);
206 av_log(avctx, AV_LOG_ERROR, "Failed to initialize MPP context (code = %d).\n", ret);
207 ret = AVERROR_UNKNOWN;
211 // make decode calls blocking with a timeout
212 paramS32 = MPP_POLL_BLOCK;
213 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK, ¶mS32);
215 av_log(avctx, AV_LOG_ERROR, "Failed to set blocking mode on MPI (code = %d).\n", ret);
216 ret = AVERROR_UNKNOWN;
220 paramS64 = RECEIVE_FRAME_TIMEOUT;
221 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK_TIMEOUT, ¶mS64);
223 av_log(avctx, AV_LOG_ERROR, "Failed to set block timeout on MPI (code = %d).\n", ret);
224 ret = AVERROR_UNKNOWN;
228 ret = mpp_buffer_group_get_internal(&decoder->frame_group, MPP_BUFFER_TYPE_ION);
230 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve buffer group (code = %d)\n", ret);
231 ret = AVERROR_UNKNOWN;
235 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_SET_EXT_BUF_GROUP, decoder->frame_group);
237 av_log(avctx, AV_LOG_ERROR, "Failed to assign buffer group (code = %d)\n", ret);
238 ret = AVERROR_UNKNOWN;
242 ret = mpp_buffer_group_limit_config(decoder->frame_group, 0, FRAMEGROUP_MAX_FRAMES);
244 av_log(avctx, AV_LOG_ERROR, "Failed to set buffer group limit (code = %d)\n", ret);
245 ret = AVERROR_UNKNOWN;
249 decoder->first_packet = 1;
251 av_log(avctx, AV_LOG_DEBUG, "RKMPP decoder initialized successfully.\n");
253 decoder->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM);
254 if (!decoder->device_ref) {
255 ret = AVERROR(ENOMEM);
258 ret = av_hwdevice_ctx_init(decoder->device_ref);
265 av_log(avctx, AV_LOG_ERROR, "Failed to initialize RKMPP decoder.\n");
266 rkmpp_close_decoder(avctx);
270 static int rkmpp_send_packet(AVCodecContext *avctx, const AVPacket *avpkt)
272 RKMPPDecodeContext *rk_context = avctx->priv_data;
273 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
278 av_log(avctx, AV_LOG_DEBUG, "End of stream.\n");
279 decoder->eos_reached = 1;
280 ret = rkmpp_write_data(avctx, NULL, 0, 0);
282 av_log(avctx, AV_LOG_ERROR, "Failed to send EOS to decoder (code = %d)\n", ret);
286 // on first packet, send extradata
287 if (decoder->first_packet) {
288 if (avctx->extradata_size) {
289 ret = rkmpp_write_data(avctx, avctx->extradata,
290 avctx->extradata_size,
293 av_log(avctx, AV_LOG_ERROR, "Failed to write extradata to decoder (code = %d)\n", ret);
297 decoder->first_packet = 0;
301 ret = rkmpp_write_data(avctx, avpkt->data, avpkt->size, avpkt->pts);
302 if (ret && ret!=AVERROR(EAGAIN))
303 av_log(avctx, AV_LOG_ERROR, "Failed to write data to decoder (code = %d)\n", ret);
308 static void rkmpp_release_frame(void *opaque, uint8_t *data)
310 AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)data;
311 AVBufferRef *framecontextref = (AVBufferRef *)opaque;
312 RKMPPFrameContext *framecontext = (RKMPPFrameContext *)framecontextref->data;
314 mpp_frame_deinit(&framecontext->frame);
315 av_buffer_unref(&framecontext->decoder_ref);
316 av_buffer_unref(&framecontextref);
321 static int rkmpp_retrieve_frame(AVCodecContext *avctx, AVFrame *frame)
323 RKMPPDecodeContext *rk_context = avctx->priv_data;
324 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
325 RKMPPFrameContext *framecontext = NULL;
326 AVBufferRef *framecontextref = NULL;
328 MppFrame mppframe = NULL;
329 MppBuffer buffer = NULL;
330 AVDRMFrameDescriptor *desc = NULL;
331 AVDRMLayerDescriptor *layer = NULL;
334 MppFrameFormat mppformat;
337 // on start of decoding, MPP can return -1, which is supposed to be expected
338 // this is due to some internal MPP init which is not completed, that will
339 // only happen in the first few frames queries, but should not be interpreted
340 // as an error, Therefore we need to retry a couple times when we get -1
341 // in order to let it time to complete it's init, then we sleep a bit between retries.
343 ret = decoder->mpi->decode_get_frame(decoder->ctx, &mppframe);
344 if (ret != MPP_OK && ret != MPP_ERR_TIMEOUT && !decoder->first_frame) {
345 if (retrycount < 5) {
346 av_log(avctx, AV_LOG_DEBUG, "Failed to get a frame, retrying (code = %d, retrycount = %d)\n", ret, retrycount);
349 goto retry_get_frame;
351 av_log(avctx, AV_LOG_ERROR, "Failed to get a frame from MPP (code = %d)\n", ret);
357 // Check whether we have a special frame or not
358 if (mpp_frame_get_info_change(mppframe)) {
359 AVHWFramesContext *hwframes;
361 av_log(avctx, AV_LOG_INFO, "Decoder noticed an info change (%dx%d), format=%d\n",
362 (int)mpp_frame_get_width(mppframe), (int)mpp_frame_get_height(mppframe),
363 (int)mpp_frame_get_fmt(mppframe));
365 avctx->width = mpp_frame_get_width(mppframe);
366 avctx->height = mpp_frame_get_height(mppframe);
368 decoder->mpi->control(decoder->ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
369 decoder->first_frame = 1;
371 av_buffer_unref(&decoder->frames_ref);
373 decoder->frames_ref = av_hwframe_ctx_alloc(decoder->device_ref);
374 if (!decoder->frames_ref) {
375 ret = AVERROR(ENOMEM);
379 mppformat = mpp_frame_get_fmt(mppframe);
380 drmformat = rkmpp_get_frameformat(mppformat);
382 hwframes = (AVHWFramesContext*)decoder->frames_ref->data;
383 hwframes->format = AV_PIX_FMT_DRM_PRIME;
384 hwframes->sw_format = drmformat == DRM_FORMAT_NV12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NONE;
385 hwframes->width = avctx->width;
386 hwframes->height = avctx->height;
387 ret = av_hwframe_ctx_init(decoder->frames_ref);
391 // here decoder is fully initialized, we need to feed it again with data
392 ret = AVERROR(EAGAIN);
394 } else if (mpp_frame_get_eos(mppframe)) {
395 av_log(avctx, AV_LOG_DEBUG, "Received a EOS frame.\n");
396 decoder->eos_reached = 1;
399 } else if (mpp_frame_get_discard(mppframe)) {
400 av_log(avctx, AV_LOG_DEBUG, "Received a discard frame.\n");
401 ret = AVERROR(EAGAIN);
403 } else if (mpp_frame_get_errinfo(mppframe)) {
404 av_log(avctx, AV_LOG_ERROR, "Received a errinfo frame.\n");
405 ret = AVERROR_UNKNOWN;
409 // here we should have a valid frame
410 av_log(avctx, AV_LOG_DEBUG, "Received a frame.\n");
412 // setup general frame fields
413 frame->format = AV_PIX_FMT_DRM_PRIME;
414 frame->width = mpp_frame_get_width(mppframe);
415 frame->height = mpp_frame_get_height(mppframe);
416 frame->pts = mpp_frame_get_pts(mppframe);
417 frame->color_range = mpp_frame_get_color_range(mppframe);
418 frame->color_primaries = mpp_frame_get_color_primaries(mppframe);
419 frame->color_trc = mpp_frame_get_color_trc(mppframe);
420 frame->colorspace = mpp_frame_get_colorspace(mppframe);
422 mode = mpp_frame_get_mode(mppframe);
423 frame->interlaced_frame = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_DEINTERLACED);
424 frame->top_field_first = ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_TOP_FIRST);
426 mppformat = mpp_frame_get_fmt(mppframe);
427 drmformat = rkmpp_get_frameformat(mppformat);
429 // now setup the frame buffer info
430 buffer = mpp_frame_get_buffer(mppframe);
432 desc = av_mallocz(sizeof(AVDRMFrameDescriptor));
434 ret = AVERROR(ENOMEM);
438 desc->nb_objects = 1;
439 desc->objects[0].fd = mpp_buffer_get_fd(buffer);
440 desc->objects[0].size = mpp_buffer_get_size(buffer);
443 layer = &desc->layers[0];
444 layer->format = drmformat;
445 layer->nb_planes = 2;
447 layer->planes[0].object_index = 0;
448 layer->planes[0].offset = 0;
449 layer->planes[0].pitch = mpp_frame_get_hor_stride(mppframe);
451 layer->planes[1].object_index = 0;
452 layer->planes[1].offset = layer->planes[0].pitch * mpp_frame_get_ver_stride(mppframe);
453 layer->planes[1].pitch = layer->planes[0].pitch;
455 // we also allocate a struct in buf[0] that will allow to hold additionnal information
456 // for releasing properly MPP frames and decoder
457 framecontextref = av_buffer_allocz(sizeof(*framecontext));
458 if (!framecontextref) {
459 ret = AVERROR(ENOMEM);
463 // MPP decoder needs to be closed only when all frames have been released.
464 framecontext = (RKMPPFrameContext *)framecontextref->data;
465 framecontext->decoder_ref = av_buffer_ref(rk_context->decoder_ref);
466 framecontext->frame = mppframe;
468 frame->data[0] = (uint8_t *)desc;
469 frame->buf[0] = av_buffer_create((uint8_t *)desc, sizeof(*desc), rkmpp_release_frame,
470 framecontextref, AV_BUFFER_FLAG_READONLY);
472 if (!frame->buf[0]) {
473 ret = AVERROR(ENOMEM);
477 frame->hw_frames_ctx = av_buffer_ref(decoder->frames_ref);
478 if (!frame->hw_frames_ctx) {
479 ret = AVERROR(ENOMEM);
483 decoder->first_frame = 0;
486 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret);
487 mpp_frame_deinit(&mppframe);
489 } else if (decoder->eos_reached) {
491 } else if (ret == MPP_ERR_TIMEOUT) {
492 av_log(avctx, AV_LOG_DEBUG, "Timeout when trying to get a frame from MPP\n");
495 return AVERROR(EAGAIN);
499 mpp_frame_deinit(&mppframe);
502 av_buffer_unref(&framecontext->decoder_ref);
505 av_buffer_unref(&framecontextref);
513 static int rkmpp_receive_frame(AVCodecContext *avctx, AVFrame *frame)
515 RKMPPDecodeContext *rk_context = avctx->priv_data;
516 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
519 RK_S32 usedslots, freeslots;
521 if (!decoder->eos_reached) {
522 // we get the available slots in decoder
523 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_GET_STREAM_COUNT, &usedslots);
525 av_log(avctx, AV_LOG_ERROR, "Failed to get decoder used slots (code = %d).\n", ret);
529 freeslots = INPUT_MAX_PACKETS - usedslots;
531 ret = ff_decode_get_packet(avctx, &pkt);
532 if (ret < 0 && ret != AVERROR_EOF) {
536 ret = rkmpp_send_packet(avctx, &pkt);
537 av_packet_unref(&pkt);
540 av_log(avctx, AV_LOG_ERROR, "Failed to send packet to decoder (code = %d)\n", ret);
545 // make sure we keep decoder full
547 return AVERROR(EAGAIN);
550 return rkmpp_retrieve_frame(avctx, frame);
553 static void rkmpp_flush(AVCodecContext *avctx)
555 RKMPPDecodeContext *rk_context = avctx->priv_data;
556 RKMPPDecoder *decoder = (RKMPPDecoder *)rk_context->decoder_ref->data;
559 av_log(avctx, AV_LOG_DEBUG, "Flush.\n");
561 ret = decoder->mpi->reset(decoder->ctx);
563 decoder->first_frame = 1;
564 decoder->first_packet = 1;
566 av_log(avctx, AV_LOG_ERROR, "Failed to reset MPI (code = %d)\n", ret);
569 static const AVCodecHWConfigInternal *rkmpp_hw_configs[] = {
570 HW_CONFIG_INTERNAL(DRM_PRIME),
574 #define RKMPP_DEC_CLASS(NAME) \
575 static const AVClass rkmpp_##NAME##_dec_class = { \
576 .class_name = "rkmpp_" #NAME "_dec", \
577 .version = LIBAVUTIL_VERSION_INT, \
580 #define RKMPP_DEC(NAME, ID, BSFS) \
581 RKMPP_DEC_CLASS(NAME) \
582 AVCodec ff_##NAME##_rkmpp_decoder = { \
583 .name = #NAME "_rkmpp", \
584 .long_name = NULL_IF_CONFIG_SMALL(#NAME " (rkmpp)"), \
585 .type = AVMEDIA_TYPE_VIDEO, \
587 .priv_data_size = sizeof(RKMPPDecodeContext), \
588 .init = rkmpp_init_decoder, \
589 .close = rkmpp_close_decoder, \
590 .receive_frame = rkmpp_receive_frame, \
591 .flush = rkmpp_flush, \
592 .priv_class = &rkmpp_##NAME##_dec_class, \
593 .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \
594 .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_DRM_PRIME, \
596 .hw_configs = rkmpp_hw_configs, \
598 .wrapper_name = "rkmpp", \
601 RKMPP_DEC(h264, AV_CODEC_ID_H264, "h264_mp4toannexb")
602 RKMPP_DEC(hevc, AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
603 RKMPP_DEC(vp8, AV_CODEC_ID_VP8, NULL)
604 RKMPP_DEC(vp9, AV_CODEC_ID_VP9, NULL)