avcodec/videotoolbox: pass through hevc param changes to the decoder
[ffmpeg.git] / libavcodec / videotoolbox.c
1 /*
2  * Videotoolbox hardware acceleration
3  *
4  * copyright (c) 2012 Sebastien Zwickert
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22
23 #include "config.h"
24 #include "videotoolbox.h"
25 #include "libavutil/hwcontext_videotoolbox.h"
26 #include "vt_internal.h"
27 #include "libavutil/avutil.h"
28 #include "libavutil/hwcontext.h"
29 #include "bytestream.h"
30 #include "h264dec.h"
31 #include "hevcdec.h"
32 #include "mpegvideo.h"
33 #include <TargetConditionals.h>
34
35 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
36 #  define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
37 #endif
38
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42
43 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING  12
44
45 static void videotoolbox_stop(AVCodecContext *avctx);
46 static int videotoolbox_start(AVCodecContext *avctx);
47
48 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
49 {
50     CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
51     CVPixelBufferRelease(cv_buffer);
52 }
53
54 static int videotoolbox_buffer_copy(VTContext *vtctx,
55                                     const uint8_t *buffer,
56                                     uint32_t size)
57 {
58     void *tmp;
59
60     tmp = av_fast_realloc(vtctx->bitstream,
61                          &vtctx->allocated_size,
62                          size);
63
64     if (!tmp)
65         return AVERROR(ENOMEM);
66
67     vtctx->bitstream = tmp;
68     memcpy(vtctx->bitstream, buffer, size);
69     vtctx->bitstream_size = size;
70
71     return 0;
72 }
73
74 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
75 {
76     frame->width  = avctx->width;
77     frame->height = avctx->height;
78     frame->format = avctx->pix_fmt;
79     frame->buf[0] = av_buffer_alloc(1);
80
81     if (!frame->buf[0])
82         return AVERROR(ENOMEM);
83
84     return 0;
85 }
86
87 #define AV_W8(p, v) *(p) = (v)
88
89 CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
90 {
91     H264Context *h     = avctx->priv_data;
92     CFDataRef data = NULL;
93     uint8_t *p;
94     int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size;
95     uint8_t *vt_extradata = av_malloc(vt_extradata_size);
96     if (!vt_extradata)
97         return NULL;
98
99     p = vt_extradata;
100
101     AV_W8(p + 0, 1); /* version */
102     AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
103     AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
104     AV_W8(p + 3, h->ps.sps->data[3]); /* level */
105     AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
106     AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
107     AV_WB16(p + 6, h->ps.sps->data_size);
108     memcpy(p + 8, h->ps.sps->data, h->ps.sps->data_size);
109     p += 8 + h->ps.sps->data_size;
110     AV_W8(p + 0, 1); /* number of pps */
111     AV_WB16(p + 1, h->ps.pps->data_size);
112     memcpy(p + 3, h->ps.pps->data, h->ps.pps->data_size);
113
114     p += 3 + h->ps.pps->data_size;
115     av_assert0(p - vt_extradata == vt_extradata_size);
116
117     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
118     av_free(vt_extradata);
119     return data;
120 }
121
122 CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
123 {
124     HEVCContext *h = avctx->priv_data;
125     const HEVCVPS *vps = (const HEVCVPS *)h->ps.vps_list[0]->data;
126     const HEVCSPS *sps = (const HEVCSPS *)h->ps.sps_list[0]->data;
127     int i, num_pps = 0;
128     const HEVCPPS *pps = h->ps.pps;
129     PTLCommon ptlc = vps->ptl.general_ptl;
130     VUI vui = sps->vui;
131     uint8_t parallelismType;
132     CFDataRef data = NULL;
133     uint8_t *p;
134     int vt_extradata_size = 23 + 5 + vps->data_size + 5 + sps->data_size + 3;
135     uint8_t *vt_extradata;
136
137     for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
138         if (h->ps.pps_list[i]) {
139             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
140             vt_extradata_size += 2 + pps->data_size;
141             num_pps++;
142         }
143     }
144
145     vt_extradata = av_malloc(vt_extradata_size);
146     if (!vt_extradata)
147         return NULL;
148     p = vt_extradata;
149
150     /* unsigned int(8) configurationVersion = 1; */
151     AV_W8(p + 0, 1);
152
153     /*
154      * unsigned int(2) general_profile_space;
155      * unsigned int(1) general_tier_flag;
156      * unsigned int(5) general_profile_idc;
157      */
158     AV_W8(p + 1, ptlc.profile_space << 6 |
159                  ptlc.tier_flag     << 5 |
160                  ptlc.profile_idc);
161
162     /* unsigned int(32) general_profile_compatibility_flags; */
163     memcpy(p + 2, ptlc.profile_compatibility_flag, 4);
164
165     /* unsigned int(48) general_constraint_indicator_flags; */
166     AV_W8(p + 6, ptlc.progressive_source_flag    << 7 |
167                  ptlc.interlaced_source_flag     << 6 |
168                  ptlc.non_packed_constraint_flag << 5 |
169                  ptlc.frame_only_constraint_flag << 4);
170     AV_W8(p + 7, 0);
171     AV_WN32(p + 8, 0);
172
173     /* unsigned int(8) general_level_idc; */
174     AV_W8(p + 12, ptlc.level_idc);
175
176     /*
177      * bit(4) reserved = ‘1111’b;
178      * unsigned int(12) min_spatial_segmentation_idc;
179      */
180     AV_W8(p + 13, 0xf0 | (vui.min_spatial_segmentation_idc >> 4));
181     AV_W8(p + 14, vui.min_spatial_segmentation_idc & 0xff);
182
183     /*
184      * bit(6) reserved = ‘111111’b;
185      * unsigned int(2) parallelismType;
186      */
187     if (!vui.min_spatial_segmentation_idc)
188         parallelismType = 0;
189     else if (pps->entropy_coding_sync_enabled_flag && pps->tiles_enabled_flag)
190         parallelismType = 0;
191     else if (pps->entropy_coding_sync_enabled_flag)
192         parallelismType = 3;
193     else if (pps->tiles_enabled_flag)
194         parallelismType = 2;
195     else
196         parallelismType = 1;
197     AV_W8(p + 15, 0xfc | parallelismType);
198
199     /*
200      * bit(6) reserved = ‘111111’b;
201      * unsigned int(2) chromaFormat;
202      */
203     AV_W8(p + 16, sps->chroma_format_idc | 0xfc);
204
205     /*
206      * bit(5) reserved = ‘11111’b;
207      * unsigned int(3) bitDepthLumaMinus8;
208      */
209     AV_W8(p + 17, (sps->bit_depth - 8) | 0xfc);
210
211     /*
212      * bit(5) reserved = ‘11111’b;
213      * unsigned int(3) bitDepthChromaMinus8;
214      */
215     AV_W8(p + 18, (sps->bit_depth_chroma - 8) | 0xfc);
216
217     /* bit(16) avgFrameRate; */
218     AV_WB16(p + 19, 0);
219
220     /*
221      * bit(2) constantFrameRate;
222      * bit(3) numTemporalLayers;
223      * bit(1) temporalIdNested;
224      * unsigned int(2) lengthSizeMinusOne;
225      */
226     AV_W8(p + 21, 0                             << 6 |
227                   sps->max_sub_layers           << 3 |
228                   sps->temporal_id_nesting_flag << 2 |
229                   3);
230
231     /* unsigned int(8) numOfArrays; */
232     AV_W8(p + 22, 3);
233
234     p += 23;
235     /* vps */
236     /*
237      * bit(1) array_completeness;
238      * unsigned int(1) reserved = 0;
239      * unsigned int(6) NAL_unit_type;
240      */
241     AV_W8(p, 1 << 7 |
242              HEVC_NAL_VPS & 0x3f);
243     /* unsigned int(16) numNalus; */
244     AV_WB16(p + 1, 1);
245     /* unsigned int(16) nalUnitLength; */
246     AV_WB16(p + 3, vps->data_size);
247     /* bit(8*nalUnitLength) nalUnit; */
248     memcpy(p + 5, vps->data, vps->data_size);
249     p += 5 + vps->data_size;
250
251     /* sps */
252     AV_W8(p, 1 << 7 |
253              HEVC_NAL_SPS & 0x3f);
254     AV_WB16(p + 1, 1);
255     AV_WB16(p + 3, sps->data_size);
256     memcpy(p + 5, sps->data, sps->data_size);
257     p += 5 + sps->data_size;
258
259     /* pps */
260     AV_W8(p, 1 << 7 |
261              HEVC_NAL_PPS & 0x3f);
262     AV_WB16(p + 1, num_pps);
263     p += 3;
264     for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
265         if (h->ps.pps_list[i]) {
266             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
267             AV_WB16(p, pps->data_size);
268             memcpy(p + 2, pps->data, pps->data_size);
269             p += 2 + pps->data_size;
270         }
271     }
272
273     av_assert0(p - vt_extradata == vt_extradata_size);
274
275     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
276     av_free(vt_extradata);
277     return data;
278 }
279
280 int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
281 {
282     av_buffer_unref(&frame->buf[0]);
283
284     frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
285                                      sizeof(vtctx->frame),
286                                      videotoolbox_buffer_release,
287                                      NULL,
288                                      AV_BUFFER_FLAG_READONLY);
289     if (!frame->buf[0]) {
290         return AVERROR(ENOMEM);
291     }
292
293     frame->data[3] = (uint8_t*)vtctx->frame;
294     vtctx->frame = NULL;
295
296     return 0;
297 }
298
299 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
300                                      const uint8_t *buffer,
301                                      uint32_t size)
302 {
303     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
304     H264Context *h  = avctx->priv_data;
305
306     if (h->is_avc == 1) {
307         return videotoolbox_buffer_copy(vtctx, buffer, size);
308     }
309
310     return 0;
311 }
312
313 static int videotoolbox_h264_decode_params(AVCodecContext *avctx,
314                                            int type,
315                                            const uint8_t *buffer,
316                                            uint32_t size)
317 {
318     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
319
320     if (type == H264_NAL_SPS) {
321         if (!vtctx->sps || vtctx->sps_len != size || memcmp(buffer, vtctx->sps, size) != 0) {
322             vtctx->sps = av_fast_realloc(vtctx->sps, &vtctx->sps_capa, size);
323             if (vtctx->sps)
324                 memcpy(vtctx->sps, buffer, size);
325             vtctx->reconfig_needed = true;
326             vtctx->sps_len = size;
327         }
328     }
329
330     // pass-through new PPS to the decoder
331     return ff_videotoolbox_h264_decode_slice(avctx, buffer, size);
332 }
333
334 int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
335                                       const uint8_t *buffer,
336                                       uint32_t size)
337 {
338     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
339     H264Context *h  = avctx->priv_data;
340     void *tmp;
341
342     if (h->is_avc == 1)
343         return 0;
344
345     tmp = av_fast_realloc(vtctx->bitstream,
346                           &vtctx->allocated_size,
347                           vtctx->bitstream_size+size+4);
348     if (!tmp)
349         return AVERROR(ENOMEM);
350
351     vtctx->bitstream = tmp;
352
353     AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
354     memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
355
356     vtctx->bitstream_size += size + 4;
357
358     return 0;
359 }
360
361 int ff_videotoolbox_uninit(AVCodecContext *avctx)
362 {
363     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
364     if (vtctx) {
365         av_freep(&vtctx->bitstream);
366         av_freep(&vtctx->sps);
367         if (vtctx->frame)
368             CVPixelBufferRelease(vtctx->frame);
369     }
370
371     return 0;
372 }
373
374 #if CONFIG_VIDEOTOOLBOX
375 // Return the AVVideotoolboxContext that matters currently. Where it comes from
376 // depends on the API used.
377 static AVVideotoolboxContext *videotoolbox_get_context(AVCodecContext *avctx)
378 {
379     // Somewhat tricky because the user can call av_videotoolbox_default_free()
380     // at any time, even when the codec is closed.
381     if (avctx->internal && avctx->internal->hwaccel_priv_data) {
382         VTContext *vtctx = avctx->internal->hwaccel_priv_data;
383         if (vtctx->vt_ctx)
384             return vtctx->vt_ctx;
385     }
386     return avctx->hwaccel_context;
387 }
388
389 static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
390 {
391     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
392     CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->frame;
393     OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
394     enum AVPixelFormat sw_format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
395     int width = CVPixelBufferGetWidth(pixbuf);
396     int height = CVPixelBufferGetHeight(pixbuf);
397     AVHWFramesContext *cached_frames;
398     int ret;
399
400     ret = ff_videotoolbox_buffer_create(vtctx, frame);
401     if (ret < 0)
402         return ret;
403
404     // Old API code path.
405     if (!vtctx->cached_hw_frames_ctx)
406         return 0;
407
408     cached_frames = (AVHWFramesContext*)vtctx->cached_hw_frames_ctx->data;
409
410     if (cached_frames->sw_format != sw_format ||
411         cached_frames->width != width ||
412         cached_frames->height != height) {
413         AVBufferRef *hw_frames_ctx = av_hwframe_ctx_alloc(cached_frames->device_ref);
414         AVHWFramesContext *hw_frames;
415         if (!hw_frames_ctx)
416             return AVERROR(ENOMEM);
417
418         hw_frames = (AVHWFramesContext*)hw_frames_ctx->data;
419         hw_frames->format = cached_frames->format;
420         hw_frames->sw_format = sw_format;
421         hw_frames->width = width;
422         hw_frames->height = height;
423
424         ret = av_hwframe_ctx_init(hw_frames_ctx);
425         if (ret < 0) {
426             av_buffer_unref(&hw_frames_ctx);
427             return ret;
428         }
429
430         av_buffer_unref(&vtctx->cached_hw_frames_ctx);
431         vtctx->cached_hw_frames_ctx = hw_frames_ctx;
432     }
433
434     av_buffer_unref(&frame->hw_frames_ctx);
435     frame->hw_frames_ctx = av_buffer_ref(vtctx->cached_hw_frames_ctx);
436     if (!frame->hw_frames_ctx)
437         return AVERROR(ENOMEM);
438
439     return 0;
440 }
441
442 static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
443 {
444     int i;
445     uint8_t b;
446
447     for (i = 3; i >= 0; i--) {
448         b = (length >> (i * 7)) & 0x7F;
449         if (i != 0)
450             b |= 0x80;
451
452         bytestream2_put_byteu(pb, b);
453     }
454 }
455
456 static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
457 {
458     CFDataRef data;
459     uint8_t *rw_extradata;
460     PutByteContext pb;
461     int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
462     // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
463     int config_size = 13 + 5 + avctx->extradata_size;
464     int s;
465
466     if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
467         return NULL;
468
469     bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
470     bytestream2_put_byteu(&pb, 0);        // version
471     bytestream2_put_ne24(&pb, 0);         // flags
472
473     // elementary stream descriptor
474     bytestream2_put_byteu(&pb, 0x03);     // ES_DescrTag
475     videotoolbox_write_mp4_descr_length(&pb, full_size);
476     bytestream2_put_ne16(&pb, 0);         // esid
477     bytestream2_put_byteu(&pb, 0);        // stream priority (0-32)
478
479     // decoder configuration descriptor
480     bytestream2_put_byteu(&pb, 0x04);     // DecoderConfigDescrTag
481     videotoolbox_write_mp4_descr_length(&pb, config_size);
482     bytestream2_put_byteu(&pb, 32);       // object type indication. 32 = AV_CODEC_ID_MPEG4
483     bytestream2_put_byteu(&pb, 0x11);     // stream type
484     bytestream2_put_ne24(&pb, 0);         // buffer size
485     bytestream2_put_ne32(&pb, 0);         // max bitrate
486     bytestream2_put_ne32(&pb, 0);         // avg bitrate
487
488     // decoder specific descriptor
489     bytestream2_put_byteu(&pb, 0x05);     ///< DecSpecificInfoTag
490     videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
491
492     bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
493
494     // SLConfigDescriptor
495     bytestream2_put_byteu(&pb, 0x06);     // SLConfigDescrTag
496     bytestream2_put_byteu(&pb, 0x01);     // length
497     bytestream2_put_byteu(&pb, 0x02);     //
498
499     s = bytestream2_size_p(&pb);
500
501     data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
502
503     av_freep(&rw_extradata);
504     return data;
505 }
506
507 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
508                                                            void *buffer,
509                                                            int size)
510 {
511     OSStatus status;
512     CMBlockBufferRef  block_buf;
513     CMSampleBufferRef sample_buf;
514
515     block_buf  = NULL;
516     sample_buf = NULL;
517
518     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
519                                                 buffer,             // memoryBlock
520                                                 size,               // blockLength
521                                                 kCFAllocatorNull,   // blockAllocator
522                                                 NULL,               // customBlockSource
523                                                 0,                  // offsetToData
524                                                 size,               // dataLength
525                                                 0,                  // flags
526                                                 &block_buf);
527
528     if (!status) {
529         status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
530                                       block_buf,            // dataBuffer
531                                       TRUE,                 // dataReady
532                                       0,                    // makeDataReadyCallback
533                                       0,                    // makeDataReadyRefcon
534                                       fmt_desc,             // formatDescription
535                                       1,                    // numSamples
536                                       0,                    // numSampleTimingEntries
537                                       NULL,                 // sampleTimingArray
538                                       0,                    // numSampleSizeEntries
539                                       NULL,                 // sampleSizeArray
540                                       &sample_buf);
541     }
542
543     if (block_buf)
544         CFRelease(block_buf);
545
546     return sample_buf;
547 }
548
549 static void videotoolbox_decoder_callback(void *opaque,
550                                           void *sourceFrameRefCon,
551                                           OSStatus status,
552                                           VTDecodeInfoFlags flags,
553                                           CVImageBufferRef image_buffer,
554                                           CMTime pts,
555                                           CMTime duration)
556 {
557     AVCodecContext *avctx = opaque;
558     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
559
560     if (vtctx->frame) {
561         CVPixelBufferRelease(vtctx->frame);
562         vtctx->frame = NULL;
563     }
564
565     if (!image_buffer) {
566         av_log(NULL, AV_LOG_DEBUG, "vt decoder cb: output image buffer is null\n");
567         return;
568     }
569
570     vtctx->frame = CVPixelBufferRetain(image_buffer);
571 }
572
573 static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
574 {
575     OSStatus status;
576     CMSampleBufferRef sample_buf;
577     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
578     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
579
580     sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
581                                                    vtctx->bitstream,
582                                                    vtctx->bitstream_size);
583
584     if (!sample_buf)
585         return -1;
586
587     status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
588                                                sample_buf,
589                                                0,       // decodeFlags
590                                                NULL,    // sourceFrameRefCon
591                                                0);      // infoFlagsOut
592     if (status == noErr)
593         status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
594
595     CFRelease(sample_buf);
596
597     return status;
598 }
599
600 static const char *videotoolbox_error_string(OSStatus status)
601 {
602     switch (status) {
603         case kVTVideoDecoderBadDataErr:
604             return "bad data";
605         case kVTVideoDecoderMalfunctionErr:
606             return "decoder malfunction";
607         case kVTInvalidSessionErr:
608             return "invalid session";
609     }
610     return "unknown";
611 }
612
613 static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
614 {
615     OSStatus status;
616     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
617     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
618
619     if (vtctx->reconfig_needed == true) {
620         vtctx->reconfig_needed = false;
621         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox decoder needs reconfig, restarting..\n");
622         videotoolbox_stop(avctx);
623         if (videotoolbox_start(avctx) != 0) {
624             return AVERROR_EXTERNAL;
625         }
626     }
627
628     if (!videotoolbox->session || !vtctx->bitstream || !vtctx->bitstream_size)
629         return AVERROR_INVALIDDATA;
630
631     status = videotoolbox_session_decode_frame(avctx);
632     if (status != noErr) {
633         if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr)
634             vtctx->reconfig_needed = true;
635         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (int)status);
636         return AVERROR_UNKNOWN;
637     }
638
639     if (!vtctx->frame) {
640         vtctx->reconfig_needed = true;
641         return AVERROR_UNKNOWN;
642     }
643
644     return videotoolbox_buffer_create(avctx, frame);
645 }
646
647 static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
648 {
649     H264Context *h = avctx->priv_data;
650     AVFrame *frame = h->cur_pic_ptr->f;
651     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
652     int ret = videotoolbox_common_end_frame(avctx, frame);
653     vtctx->bitstream_size = 0;
654     return ret;
655 }
656
657 static int videotoolbox_hevc_decode_params(AVCodecContext *avctx,
658                                            int type,
659                                            const uint8_t *buffer,
660                                            uint32_t size)
661 {
662     return ff_videotoolbox_h264_decode_slice(avctx, buffer, size);
663 }
664
665 static int videotoolbox_hevc_end_frame(AVCodecContext *avctx)
666 {
667     HEVCContext *h = avctx->priv_data;
668     AVFrame *frame = h->ref->frame;
669     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
670     int ret;
671
672     ret = videotoolbox_common_end_frame(avctx, frame);
673     vtctx->bitstream_size = 0;
674     return ret;
675 }
676
677 static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
678                                          const uint8_t *buffer,
679                                          uint32_t size)
680 {
681     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
682
683     return videotoolbox_buffer_copy(vtctx, buffer, size);
684 }
685
686 static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
687                                           const uint8_t *buffer,
688                                           uint32_t size)
689 {
690     return 0;
691 }
692
693 static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
694 {
695     MpegEncContext *s = avctx->priv_data;
696     AVFrame *frame = s->current_picture_ptr->f;
697
698     return videotoolbox_common_end_frame(avctx, frame);
699 }
700
701 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
702                                                           AVCodecContext *avctx)
703 {
704     CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
705                                                                    0,
706                                                                    &kCFTypeDictionaryKeyCallBacks,
707                                                                    &kCFTypeDictionaryValueCallBacks);
708
709     CFDictionarySetValue(config_info,
710                          kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
711                          kCFBooleanTrue);
712
713     CFMutableDictionaryRef avc_info;
714     CFDataRef data = NULL;
715
716     avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
717                                          1,
718                                          &kCFTypeDictionaryKeyCallBacks,
719                                          &kCFTypeDictionaryValueCallBacks);
720
721     switch (codec_type) {
722     case kCMVideoCodecType_MPEG4Video :
723         if (avctx->extradata_size)
724             data = videotoolbox_esds_extradata_create(avctx);
725         if (data)
726             CFDictionarySetValue(avc_info, CFSTR("esds"), data);
727         break;
728     case kCMVideoCodecType_H264 :
729         data = ff_videotoolbox_avcc_extradata_create(avctx);
730         if (data)
731             CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
732         break;
733     case kCMVideoCodecType_HEVC :
734         data = ff_videotoolbox_hvcc_extradata_create(avctx);
735         if (data)
736             CFDictionarySetValue(avc_info, CFSTR("hvcC"), data);
737         break;
738     default:
739         break;
740     }
741
742     CFDictionarySetValue(config_info,
743             kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
744             avc_info);
745
746     if (data)
747         CFRelease(data);
748
749     CFRelease(avc_info);
750     return config_info;
751 }
752
753 static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
754                                                              int height,
755                                                              OSType pix_fmt)
756 {
757     CFMutableDictionaryRef buffer_attributes;
758     CFMutableDictionaryRef io_surface_properties;
759     CFNumberRef cv_pix_fmt;
760     CFNumberRef w;
761     CFNumberRef h;
762
763     w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
764     h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
765     cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
766
767     buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
768                                                   4,
769                                                   &kCFTypeDictionaryKeyCallBacks,
770                                                   &kCFTypeDictionaryValueCallBacks);
771     io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
772                                                       0,
773                                                       &kCFTypeDictionaryKeyCallBacks,
774                                                       &kCFTypeDictionaryValueCallBacks);
775
776     if (pix_fmt)
777         CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
778     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
779     CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
780     CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
781 #if TARGET_OS_IPHONE
782     CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
783 #else
784     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
785 #endif
786
787     CFRelease(io_surface_properties);
788     CFRelease(cv_pix_fmt);
789     CFRelease(w);
790     CFRelease(h);
791
792     return buffer_attributes;
793 }
794
795 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
796                                                                    CFDictionaryRef decoder_spec,
797                                                                    int width,
798                                                                    int height)
799 {
800     CMFormatDescriptionRef cm_fmt_desc;
801     OSStatus status;
802
803     status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
804                                             codec_type,
805                                             width,
806                                             height,
807                                             decoder_spec, // Dictionary of extension
808                                             &cm_fmt_desc);
809
810     if (status)
811         return NULL;
812
813     return cm_fmt_desc;
814 }
815
816 static int videotoolbox_start(AVCodecContext *avctx)
817 {
818     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
819     OSStatus status;
820     VTDecompressionOutputCallbackRecord decoder_cb;
821     CFDictionaryRef decoder_spec;
822     CFDictionaryRef buf_attr;
823
824     if (!videotoolbox) {
825         av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
826         return -1;
827     }
828
829     switch( avctx->codec_id ) {
830     case AV_CODEC_ID_H263 :
831         videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
832         break;
833     case AV_CODEC_ID_H264 :
834         videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
835         break;
836     case AV_CODEC_ID_HEVC :
837         videotoolbox->cm_codec_type = kCMVideoCodecType_HEVC;
838         break;
839     case AV_CODEC_ID_MPEG1VIDEO :
840         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
841         break;
842     case AV_CODEC_ID_MPEG2VIDEO :
843         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
844         break;
845     case AV_CODEC_ID_MPEG4 :
846         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
847         break;
848     default :
849         break;
850     }
851
852     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
853
854     if (!decoder_spec) {
855         av_log(avctx, AV_LOG_ERROR, "decoder specification creation failed\n");
856         return -1;
857     }
858
859     videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
860                                                                 decoder_spec,
861                                                                 avctx->width,
862                                                                 avctx->height);
863     if (!videotoolbox->cm_fmt_desc) {
864         if (decoder_spec)
865             CFRelease(decoder_spec);
866
867         av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
868         return -1;
869     }
870
871     buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
872                                                      avctx->height,
873                                                      videotoolbox->cv_pix_fmt_type);
874
875     decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
876     decoder_cb.decompressionOutputRefCon   = avctx;
877
878     status = VTDecompressionSessionCreate(NULL,                      // allocator
879                                           videotoolbox->cm_fmt_desc, // videoFormatDescription
880                                           decoder_spec,              // videoDecoderSpecification
881                                           buf_attr,                  // destinationImageBufferAttributes
882                                           &decoder_cb,               // outputCallback
883                                           &videotoolbox->session);   // decompressionSessionOut
884
885     if (decoder_spec)
886         CFRelease(decoder_spec);
887     if (buf_attr)
888         CFRelease(buf_attr);
889
890     switch (status) {
891     case kVTVideoDecoderNotAvailableNowErr:
892         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox session not available.\n");
893         return AVERROR(ENOSYS);
894     case kVTVideoDecoderUnsupportedDataFormatErr:
895         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox does not support this format.\n");
896         return AVERROR(ENOSYS);
897     case kVTVideoDecoderMalfunctionErr:
898         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n");
899         return AVERROR(EINVAL);
900     case kVTVideoDecoderBadDataErr :
901         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n");
902         return AVERROR_INVALIDDATA;
903     case 0:
904         return 0;
905     default:
906         av_log(avctx, AV_LOG_VERBOSE, "Unknown VideoToolbox session creation error %u\n", (unsigned)status);
907         return AVERROR_UNKNOWN;
908     }
909 }
910
911 static void videotoolbox_stop(AVCodecContext *avctx)
912 {
913     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
914     if (!videotoolbox)
915         return;
916
917     if (videotoolbox->cm_fmt_desc) {
918         CFRelease(videotoolbox->cm_fmt_desc);
919         videotoolbox->cm_fmt_desc = NULL;
920     }
921
922     if (videotoolbox->session) {
923         VTDecompressionSessionInvalidate(videotoolbox->session);
924         CFRelease(videotoolbox->session);
925         videotoolbox->session = NULL;
926     }
927 }
928
929 static int videotoolbox_uninit(AVCodecContext *avctx)
930 {
931     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
932     if (!vtctx)
933         return 0;
934
935     ff_videotoolbox_uninit(avctx);
936
937     if (vtctx->vt_ctx)
938         videotoolbox_stop(avctx);
939
940     av_buffer_unref(&vtctx->cached_hw_frames_ctx);
941     av_freep(&vtctx->vt_ctx);
942
943     return 0;
944 }
945
946 static int videotoolbox_common_init(AVCodecContext *avctx)
947 {
948     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
949     AVHWFramesContext *hw_frames;
950     int err;
951
952     // Old API - do nothing.
953     if (avctx->hwaccel_context)
954         return 0;
955
956     if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
957         av_log(avctx, AV_LOG_ERROR,
958                "Either hw_frames_ctx or hw_device_ctx must be set.\n");
959         return AVERROR(EINVAL);
960     }
961
962     vtctx->vt_ctx = av_videotoolbox_alloc_context();
963     if (!vtctx->vt_ctx) {
964         err = AVERROR(ENOMEM);
965         goto fail;
966     }
967
968     if (avctx->hw_frames_ctx) {
969         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
970     } else {
971         avctx->hw_frames_ctx = av_hwframe_ctx_alloc(avctx->hw_device_ctx);
972         if (!avctx->hw_frames_ctx) {
973             err = AVERROR(ENOMEM);
974             goto fail;
975         }
976
977         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
978         hw_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
979         hw_frames->sw_format = AV_PIX_FMT_NV12; // same as av_videotoolbox_alloc_context()
980         hw_frames->width = avctx->width;
981         hw_frames->height = avctx->height;
982
983         err = av_hwframe_ctx_init(avctx->hw_frames_ctx);
984         if (err < 0) {
985             av_buffer_unref(&avctx->hw_frames_ctx);
986             goto fail;
987         }
988     }
989
990     vtctx->cached_hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
991     if (!vtctx->cached_hw_frames_ctx) {
992         err = AVERROR(ENOMEM);
993         goto fail;
994     }
995
996     vtctx->vt_ctx->cv_pix_fmt_type =
997         av_map_videotoolbox_format_from_pixfmt(hw_frames->sw_format);
998     if (!vtctx->vt_ctx->cv_pix_fmt_type) {
999         av_log(avctx, AV_LOG_ERROR, "Unknown sw_format.\n");
1000         err = AVERROR(EINVAL);
1001         goto fail;
1002     }
1003
1004     err = videotoolbox_start(avctx);
1005     if (err < 0)
1006         goto fail;
1007
1008     return 0;
1009
1010 fail:
1011     videotoolbox_uninit(avctx);
1012     return err;
1013 }
1014
1015 static int videotoolbox_frame_params(AVCodecContext *avctx,
1016                                      AVBufferRef *hw_frames_ctx)
1017 {
1018     AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
1019
1020     frames_ctx->format            = AV_PIX_FMT_VIDEOTOOLBOX;
1021     frames_ctx->width             = avctx->coded_width;
1022     frames_ctx->height            = avctx->coded_height;
1023     frames_ctx->sw_format         = AV_PIX_FMT_NV12;
1024
1025     return 0;
1026 }
1027
1028 AVHWAccel ff_h263_videotoolbox_hwaccel = {
1029     .name           = "h263_videotoolbox",
1030     .type           = AVMEDIA_TYPE_VIDEO,
1031     .id             = AV_CODEC_ID_H263,
1032     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1033     .alloc_frame    = ff_videotoolbox_alloc_frame,
1034     .start_frame    = videotoolbox_mpeg_start_frame,
1035     .decode_slice   = videotoolbox_mpeg_decode_slice,
1036     .end_frame      = videotoolbox_mpeg_end_frame,
1037     .frame_params   = videotoolbox_frame_params,
1038     .init           = videotoolbox_common_init,
1039     .uninit         = videotoolbox_uninit,
1040     .priv_data_size = sizeof(VTContext),
1041 };
1042
1043 AVHWAccel ff_hevc_videotoolbox_hwaccel = {
1044     .name           = "hevc_videotoolbox",
1045     .type           = AVMEDIA_TYPE_VIDEO,
1046     .id             = AV_CODEC_ID_HEVC,
1047     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1048     .alloc_frame    = ff_videotoolbox_alloc_frame,
1049     .start_frame    = ff_videotoolbox_h264_start_frame,
1050     .decode_slice   = ff_videotoolbox_h264_decode_slice,
1051     .decode_params  = videotoolbox_hevc_decode_params,
1052     .end_frame      = videotoolbox_hevc_end_frame,
1053     .frame_params   = videotoolbox_frame_params,
1054     .init           = videotoolbox_common_init,
1055     .uninit         = ff_videotoolbox_uninit,
1056     .priv_data_size = sizeof(VTContext),
1057 };
1058
1059 AVHWAccel ff_h264_videotoolbox_hwaccel = {
1060     .name           = "h264_videotoolbox",
1061     .type           = AVMEDIA_TYPE_VIDEO,
1062     .id             = AV_CODEC_ID_H264,
1063     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1064     .alloc_frame    = ff_videotoolbox_alloc_frame,
1065     .start_frame    = ff_videotoolbox_h264_start_frame,
1066     .decode_slice   = ff_videotoolbox_h264_decode_slice,
1067     .decode_params  = videotoolbox_h264_decode_params,
1068     .end_frame      = videotoolbox_h264_end_frame,
1069     .frame_params   = videotoolbox_frame_params,
1070     .init           = videotoolbox_common_init,
1071     .uninit         = videotoolbox_uninit,
1072     .priv_data_size = sizeof(VTContext),
1073 };
1074
1075 AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
1076     .name           = "mpeg1_videotoolbox",
1077     .type           = AVMEDIA_TYPE_VIDEO,
1078     .id             = AV_CODEC_ID_MPEG1VIDEO,
1079     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1080     .alloc_frame    = ff_videotoolbox_alloc_frame,
1081     .start_frame    = videotoolbox_mpeg_start_frame,
1082     .decode_slice   = videotoolbox_mpeg_decode_slice,
1083     .end_frame      = videotoolbox_mpeg_end_frame,
1084     .frame_params   = videotoolbox_frame_params,
1085     .init           = videotoolbox_common_init,
1086     .uninit         = videotoolbox_uninit,
1087     .priv_data_size = sizeof(VTContext),
1088 };
1089
1090 AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
1091     .name           = "mpeg2_videotoolbox",
1092     .type           = AVMEDIA_TYPE_VIDEO,
1093     .id             = AV_CODEC_ID_MPEG2VIDEO,
1094     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1095     .alloc_frame    = ff_videotoolbox_alloc_frame,
1096     .start_frame    = videotoolbox_mpeg_start_frame,
1097     .decode_slice   = videotoolbox_mpeg_decode_slice,
1098     .end_frame      = videotoolbox_mpeg_end_frame,
1099     .frame_params   = videotoolbox_frame_params,
1100     .init           = videotoolbox_common_init,
1101     .uninit         = videotoolbox_uninit,
1102     .priv_data_size = sizeof(VTContext),
1103 };
1104
1105 AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
1106     .name           = "mpeg4_videotoolbox",
1107     .type           = AVMEDIA_TYPE_VIDEO,
1108     .id             = AV_CODEC_ID_MPEG4,
1109     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1110     .alloc_frame    = ff_videotoolbox_alloc_frame,
1111     .start_frame    = videotoolbox_mpeg_start_frame,
1112     .decode_slice   = videotoolbox_mpeg_decode_slice,
1113     .end_frame      = videotoolbox_mpeg_end_frame,
1114     .frame_params   = videotoolbox_frame_params,
1115     .init           = videotoolbox_common_init,
1116     .uninit         = videotoolbox_uninit,
1117     .priv_data_size = sizeof(VTContext),
1118 };
1119
1120 AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
1121 {
1122     AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
1123
1124     if (ret) {
1125         ret->output_callback = videotoolbox_decoder_callback;
1126         ret->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1127     }
1128
1129     return ret;
1130 }
1131
1132 int av_videotoolbox_default_init(AVCodecContext *avctx)
1133 {
1134     return av_videotoolbox_default_init2(avctx, NULL);
1135 }
1136
1137 int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
1138 {
1139     avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context();
1140     if (!avctx->hwaccel_context)
1141         return AVERROR(ENOMEM);
1142     return videotoolbox_start(avctx);
1143 }
1144
1145 void av_videotoolbox_default_free(AVCodecContext *avctx)
1146 {
1147
1148     videotoolbox_stop(avctx);
1149     av_freep(&avctx->hwaccel_context);
1150 }
1151 #endif /* CONFIG_VIDEOTOOLBOX */