avcodec/videotoolbox: use early return in videotoolbox_default_free
[ffmpeg.git] / libavcodec / videotoolbox.c
1 /*
2  * Videotoolbox hardware acceleration
3  *
4  * copyright (c) 2012 Sebastien Zwickert
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22
23 #include "config.h"
24 #include "videotoolbox.h"
25 #include "libavutil/hwcontext_videotoolbox.h"
26 #include "vt_internal.h"
27 #include "libavutil/avutil.h"
28 #include "libavutil/hwcontext.h"
29 #include "bytestream.h"
30 #include "h264dec.h"
31 #include "hevcdec.h"
32 #include "mpegvideo.h"
33 #include <TargetConditionals.h>
34
35 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
36 #  define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
37 #endif
38
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42
43 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING  12
44
45 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
46 {
47     CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
48     CVPixelBufferRelease(cv_buffer);
49 }
50
51 static int videotoolbox_buffer_copy(VTContext *vtctx,
52                                     const uint8_t *buffer,
53                                     uint32_t size)
54 {
55     void *tmp;
56
57     tmp = av_fast_realloc(vtctx->bitstream,
58                          &vtctx->allocated_size,
59                          size);
60
61     if (!tmp)
62         return AVERROR(ENOMEM);
63
64     vtctx->bitstream = tmp;
65     memcpy(vtctx->bitstream, buffer, size);
66     vtctx->bitstream_size = size;
67
68     return 0;
69 }
70
71 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
72 {
73     frame->width  = avctx->width;
74     frame->height = avctx->height;
75     frame->format = avctx->pix_fmt;
76     frame->buf[0] = av_buffer_alloc(1);
77
78     if (!frame->buf[0])
79         return AVERROR(ENOMEM);
80
81     return 0;
82 }
83
84 #define AV_W8(p, v) *(p) = (v)
85
86 CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
87 {
88     H264Context *h     = avctx->priv_data;
89     CFDataRef data = NULL;
90     uint8_t *p;
91     int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size;
92     uint8_t *vt_extradata = av_malloc(vt_extradata_size);
93     if (!vt_extradata)
94         return NULL;
95
96     p = vt_extradata;
97
98     AV_W8(p + 0, 1); /* version */
99     AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
100     AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
101     AV_W8(p + 3, h->ps.sps->data[3]); /* level */
102     AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
103     AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
104     AV_WB16(p + 6, h->ps.sps->data_size);
105     memcpy(p + 8, h->ps.sps->data, h->ps.sps->data_size);
106     p += 8 + h->ps.sps->data_size;
107     AV_W8(p + 0, 1); /* number of pps */
108     AV_WB16(p + 1, h->ps.pps->data_size);
109     memcpy(p + 3, h->ps.pps->data, h->ps.pps->data_size);
110
111     p += 3 + h->ps.pps->data_size;
112     av_assert0(p - vt_extradata == vt_extradata_size);
113
114     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
115     av_free(vt_extradata);
116     return data;
117 }
118
119 CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
120 {
121     HEVCContext *h = avctx->priv_data;
122     const HEVCVPS *vps = (const HEVCVPS *)h->ps.vps_list[0]->data;
123     const HEVCSPS *sps = (const HEVCSPS *)h->ps.sps_list[0]->data;
124     int i, num_pps = 0;
125     const HEVCPPS *pps = h->ps.pps;
126     PTLCommon ptlc = vps->ptl.general_ptl;
127     VUI vui = sps->vui;
128     uint8_t parallelismType;
129     CFDataRef data = NULL;
130     uint8_t *p;
131     int vt_extradata_size = 23 + 5 + vps->data_size + 5 + sps->data_size + 3;
132     uint8_t *vt_extradata;
133
134     for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
135         if (h->ps.pps_list[i]) {
136             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
137             vt_extradata_size += 2 + pps->data_size;
138             num_pps++;
139         }
140     }
141
142     vt_extradata = av_malloc(vt_extradata_size);
143     if (!vt_extradata)
144         return NULL;
145     p = vt_extradata;
146
147     /* unsigned int(8) configurationVersion = 1; */
148     AV_W8(p + 0, 1);
149
150     /*
151      * unsigned int(2) general_profile_space;
152      * unsigned int(1) general_tier_flag;
153      * unsigned int(5) general_profile_idc;
154      */
155     AV_W8(p + 1, ptlc.profile_space << 6 |
156                  ptlc.tier_flag     << 5 |
157                  ptlc.profile_idc);
158
159     /* unsigned int(32) general_profile_compatibility_flags; */
160     memcpy(p + 2, ptlc.profile_compatibility_flag, 4);
161
162     /* unsigned int(48) general_constraint_indicator_flags; */
163     AV_W8(p + 6, ptlc.progressive_source_flag    << 7 |
164                  ptlc.interlaced_source_flag     << 6 |
165                  ptlc.non_packed_constraint_flag << 5 |
166                  ptlc.frame_only_constraint_flag << 4);
167     AV_W8(p + 7, 0);
168     AV_WN32(p + 8, 0);
169
170     /* unsigned int(8) general_level_idc; */
171     AV_W8(p + 12, ptlc.level_idc);
172
173     /*
174      * bit(4) reserved = ‘1111’b;
175      * unsigned int(12) min_spatial_segmentation_idc;
176      */
177     AV_W8(p + 13, 0xf0 | (vui.min_spatial_segmentation_idc >> 4));
178     AV_W8(p + 14, vui.min_spatial_segmentation_idc & 0xff);
179
180     /*
181      * bit(6) reserved = ‘111111’b;
182      * unsigned int(2) parallelismType;
183      */
184     if (!vui.min_spatial_segmentation_idc)
185         parallelismType = 0;
186     else if (pps->entropy_coding_sync_enabled_flag && pps->tiles_enabled_flag)
187         parallelismType = 0;
188     else if (pps->entropy_coding_sync_enabled_flag)
189         parallelismType = 3;
190     else if (pps->tiles_enabled_flag)
191         parallelismType = 2;
192     else
193         parallelismType = 1;
194     AV_W8(p + 15, 0xfc | parallelismType);
195
196     /*
197      * bit(6) reserved = ‘111111’b;
198      * unsigned int(2) chromaFormat;
199      */
200     AV_W8(p + 16, sps->chroma_format_idc | 0xfc);
201
202     /*
203      * bit(5) reserved = ‘11111’b;
204      * unsigned int(3) bitDepthLumaMinus8;
205      */
206     AV_W8(p + 17, (sps->bit_depth - 8) | 0xfc);
207
208     /*
209      * bit(5) reserved = ‘11111’b;
210      * unsigned int(3) bitDepthChromaMinus8;
211      */
212     AV_W8(p + 18, (sps->bit_depth_chroma - 8) | 0xfc);
213
214     /* bit(16) avgFrameRate; */
215     AV_WB16(p + 19, 0);
216
217     /*
218      * bit(2) constantFrameRate;
219      * bit(3) numTemporalLayers;
220      * bit(1) temporalIdNested;
221      * unsigned int(2) lengthSizeMinusOne;
222      */
223     AV_W8(p + 21, 0                             << 6 |
224                   sps->max_sub_layers           << 3 |
225                   sps->temporal_id_nesting_flag << 2 |
226                   3);
227
228     /* unsigned int(8) numOfArrays; */
229     AV_W8(p + 22, 3);
230
231     p += 23;
232     /* vps */
233     /*
234      * bit(1) array_completeness;
235      * unsigned int(1) reserved = 0;
236      * unsigned int(6) NAL_unit_type;
237      */
238     AV_W8(p, 1 << 7 |
239              HEVC_NAL_VPS & 0x3f);
240     /* unsigned int(16) numNalus; */
241     AV_WB16(p + 1, 1);
242     /* unsigned int(16) nalUnitLength; */
243     AV_WB16(p + 3, vps->data_size);
244     /* bit(8*nalUnitLength) nalUnit; */
245     memcpy(p + 5, vps->data, vps->data_size);
246     p += 5 + vps->data_size;
247
248     /* sps */
249     AV_W8(p, 1 << 7 |
250              HEVC_NAL_SPS & 0x3f);
251     AV_WB16(p + 1, 1);
252     AV_WB16(p + 3, sps->data_size);
253     memcpy(p + 5, sps->data, sps->data_size);
254     p += 5 + sps->data_size;
255
256     /* pps */
257     AV_W8(p, 1 << 7 |
258              HEVC_NAL_PPS & 0x3f);
259     AV_WB16(p + 1, num_pps);
260     p += 3;
261     for (i = 0; i < HEVC_MAX_PPS_COUNT; i++) {
262         if (h->ps.pps_list[i]) {
263             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
264             AV_WB16(p, pps->data_size);
265             memcpy(p + 2, pps->data, pps->data_size);
266             p += 2 + pps->data_size;
267         }
268     }
269
270     av_assert0(p - vt_extradata == vt_extradata_size);
271
272     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
273     av_free(vt_extradata);
274     return data;
275 }
276
277 int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
278 {
279     av_buffer_unref(&frame->buf[0]);
280
281     frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
282                                      sizeof(vtctx->frame),
283                                      videotoolbox_buffer_release,
284                                      NULL,
285                                      AV_BUFFER_FLAG_READONLY);
286     if (!frame->buf[0]) {
287         return AVERROR(ENOMEM);
288     }
289
290     frame->data[3] = (uint8_t*)vtctx->frame;
291     vtctx->frame = NULL;
292
293     return 0;
294 }
295
296 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
297                                      const uint8_t *buffer,
298                                      uint32_t size)
299 {
300     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
301     H264Context *h  = avctx->priv_data;
302
303     vtctx->bitstream_size = 0;
304
305     if (h->is_avc == 1) {
306         return videotoolbox_buffer_copy(vtctx, buffer, size);
307     }
308
309     return 0;
310 }
311
312 int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
313                                       const uint8_t *buffer,
314                                       uint32_t size)
315 {
316     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
317     H264Context *h  = avctx->priv_data;
318     void *tmp;
319
320     if (h->is_avc == 1)
321         return 0;
322
323     tmp = av_fast_realloc(vtctx->bitstream,
324                           &vtctx->allocated_size,
325                           vtctx->bitstream_size+size+4);
326     if (!tmp)
327         return AVERROR(ENOMEM);
328
329     vtctx->bitstream = tmp;
330
331     AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
332     memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
333
334     vtctx->bitstream_size += size + 4;
335
336     return 0;
337 }
338
339 int ff_videotoolbox_uninit(AVCodecContext *avctx)
340 {
341     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
342     if (vtctx) {
343         av_freep(&vtctx->bitstream);
344         if (vtctx->frame)
345             CVPixelBufferRelease(vtctx->frame);
346     }
347
348     return 0;
349 }
350
351 #if CONFIG_VIDEOTOOLBOX
352 // Return the AVVideotoolboxContext that matters currently. Where it comes from
353 // depends on the API used.
354 static AVVideotoolboxContext *videotoolbox_get_context(AVCodecContext *avctx)
355 {
356     // Somewhat tricky because the user can call av_videotoolbox_default_free()
357     // at any time, even when the codec is closed.
358     if (avctx->internal && avctx->internal->hwaccel_priv_data) {
359         VTContext *vtctx = avctx->internal->hwaccel_priv_data;
360         if (vtctx->vt_ctx)
361             return vtctx->vt_ctx;
362     }
363     return avctx->hwaccel_context;
364 }
365
366 static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
367 {
368     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
369     CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->frame;
370     OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
371     enum AVPixelFormat sw_format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
372     int width = CVPixelBufferGetWidth(pixbuf);
373     int height = CVPixelBufferGetHeight(pixbuf);
374     AVHWFramesContext *cached_frames;
375     int ret;
376
377     ret = ff_videotoolbox_buffer_create(vtctx, frame);
378     if (ret < 0)
379         return ret;
380
381     // Old API code path.
382     if (!vtctx->cached_hw_frames_ctx)
383         return 0;
384
385     cached_frames = (AVHWFramesContext*)vtctx->cached_hw_frames_ctx->data;
386
387     if (cached_frames->sw_format != sw_format ||
388         cached_frames->width != width ||
389         cached_frames->height != height) {
390         AVBufferRef *hw_frames_ctx = av_hwframe_ctx_alloc(cached_frames->device_ref);
391         AVHWFramesContext *hw_frames;
392         if (!hw_frames_ctx)
393             return AVERROR(ENOMEM);
394
395         hw_frames = (AVHWFramesContext*)hw_frames_ctx->data;
396         hw_frames->format = cached_frames->format;
397         hw_frames->sw_format = sw_format;
398         hw_frames->width = width;
399         hw_frames->height = height;
400
401         ret = av_hwframe_ctx_init(hw_frames_ctx);
402         if (ret < 0) {
403             av_buffer_unref(&hw_frames_ctx);
404             return ret;
405         }
406
407         av_buffer_unref(&vtctx->cached_hw_frames_ctx);
408         vtctx->cached_hw_frames_ctx = hw_frames_ctx;
409     }
410
411     av_buffer_unref(&frame->hw_frames_ctx);
412     frame->hw_frames_ctx = av_buffer_ref(vtctx->cached_hw_frames_ctx);
413     if (!frame->hw_frames_ctx)
414         return AVERROR(ENOMEM);
415
416     return 0;
417 }
418
419 static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
420 {
421     int i;
422     uint8_t b;
423
424     for (i = 3; i >= 0; i--) {
425         b = (length >> (i * 7)) & 0x7F;
426         if (i != 0)
427             b |= 0x80;
428
429         bytestream2_put_byteu(pb, b);
430     }
431 }
432
433 static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
434 {
435     CFDataRef data;
436     uint8_t *rw_extradata;
437     PutByteContext pb;
438     int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
439     // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
440     int config_size = 13 + 5 + avctx->extradata_size;
441     int s;
442
443     if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
444         return NULL;
445
446     bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
447     bytestream2_put_byteu(&pb, 0);        // version
448     bytestream2_put_ne24(&pb, 0);         // flags
449
450     // elementary stream descriptor
451     bytestream2_put_byteu(&pb, 0x03);     // ES_DescrTag
452     videotoolbox_write_mp4_descr_length(&pb, full_size);
453     bytestream2_put_ne16(&pb, 0);         // esid
454     bytestream2_put_byteu(&pb, 0);        // stream priority (0-32)
455
456     // decoder configuration descriptor
457     bytestream2_put_byteu(&pb, 0x04);     // DecoderConfigDescrTag
458     videotoolbox_write_mp4_descr_length(&pb, config_size);
459     bytestream2_put_byteu(&pb, 32);       // object type indication. 32 = AV_CODEC_ID_MPEG4
460     bytestream2_put_byteu(&pb, 0x11);     // stream type
461     bytestream2_put_ne24(&pb, 0);         // buffer size
462     bytestream2_put_ne32(&pb, 0);         // max bitrate
463     bytestream2_put_ne32(&pb, 0);         // avg bitrate
464
465     // decoder specific descriptor
466     bytestream2_put_byteu(&pb, 0x05);     ///< DecSpecificInfoTag
467     videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
468
469     bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
470
471     // SLConfigDescriptor
472     bytestream2_put_byteu(&pb, 0x06);     // SLConfigDescrTag
473     bytestream2_put_byteu(&pb, 0x01);     // length
474     bytestream2_put_byteu(&pb, 0x02);     //
475
476     s = bytestream2_size_p(&pb);
477
478     data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
479
480     av_freep(&rw_extradata);
481     return data;
482 }
483
484 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
485                                                            void *buffer,
486                                                            int size)
487 {
488     OSStatus status;
489     CMBlockBufferRef  block_buf;
490     CMSampleBufferRef sample_buf;
491
492     block_buf  = NULL;
493     sample_buf = NULL;
494
495     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
496                                                 buffer,             // memoryBlock
497                                                 size,               // blockLength
498                                                 kCFAllocatorNull,   // blockAllocator
499                                                 NULL,               // customBlockSource
500                                                 0,                  // offsetToData
501                                                 size,               // dataLength
502                                                 0,                  // flags
503                                                 &block_buf);
504
505     if (!status) {
506         status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
507                                       block_buf,            // dataBuffer
508                                       TRUE,                 // dataReady
509                                       0,                    // makeDataReadyCallback
510                                       0,                    // makeDataReadyRefcon
511                                       fmt_desc,             // formatDescription
512                                       1,                    // numSamples
513                                       0,                    // numSampleTimingEntries
514                                       NULL,                 // sampleTimingArray
515                                       0,                    // numSampleSizeEntries
516                                       NULL,                 // sampleSizeArray
517                                       &sample_buf);
518     }
519
520     if (block_buf)
521         CFRelease(block_buf);
522
523     return sample_buf;
524 }
525
526 static void videotoolbox_decoder_callback(void *opaque,
527                                           void *sourceFrameRefCon,
528                                           OSStatus status,
529                                           VTDecodeInfoFlags flags,
530                                           CVImageBufferRef image_buffer,
531                                           CMTime pts,
532                                           CMTime duration)
533 {
534     AVCodecContext *avctx = opaque;
535     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
536
537     if (vtctx->frame) {
538         CVPixelBufferRelease(vtctx->frame);
539         vtctx->frame = NULL;
540     }
541
542     if (!image_buffer) {
543         av_log(NULL, AV_LOG_DEBUG, "vt decoder cb: output image buffer is null\n");
544         return;
545     }
546
547     vtctx->frame = CVPixelBufferRetain(image_buffer);
548 }
549
550 static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
551 {
552     OSStatus status;
553     CMSampleBufferRef sample_buf;
554     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
555     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
556
557     sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
558                                                    vtctx->bitstream,
559                                                    vtctx->bitstream_size);
560
561     if (!sample_buf)
562         return -1;
563
564     status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
565                                                sample_buf,
566                                                0,       // decodeFlags
567                                                NULL,    // sourceFrameRefCon
568                                                0);      // infoFlagsOut
569     if (status == noErr)
570         status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
571
572     CFRelease(sample_buf);
573
574     return status;
575 }
576
577 static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
578 {
579     int status;
580     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
581     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
582
583     if (!videotoolbox->session || !vtctx->bitstream)
584         return AVERROR_INVALIDDATA;
585
586     status = videotoolbox_session_decode_frame(avctx);
587
588     if (status) {
589         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
590         return AVERROR_UNKNOWN;
591     }
592
593     if (!vtctx->frame)
594         return AVERROR_UNKNOWN;
595
596     return videotoolbox_buffer_create(avctx, frame);
597 }
598
599 static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
600 {
601     H264Context *h = avctx->priv_data;
602     AVFrame *frame = h->cur_pic_ptr->f;
603
604     return videotoolbox_common_end_frame(avctx, frame);
605 }
606
607 static int videotoolbox_hevc_end_frame(AVCodecContext *avctx)
608 {
609     HEVCContext *h = avctx->priv_data;
610     AVFrame *frame = h->ref->frame;
611     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
612     int ret;
613
614     ret = videotoolbox_common_end_frame(avctx, frame);
615     vtctx->bitstream_size = 0;
616     return ret;
617 }
618
619 static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
620                                          const uint8_t *buffer,
621                                          uint32_t size)
622 {
623     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
624
625     return videotoolbox_buffer_copy(vtctx, buffer, size);
626 }
627
628 static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
629                                           const uint8_t *buffer,
630                                           uint32_t size)
631 {
632     return 0;
633 }
634
635 static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
636 {
637     MpegEncContext *s = avctx->priv_data;
638     AVFrame *frame = s->current_picture_ptr->f;
639
640     return videotoolbox_common_end_frame(avctx, frame);
641 }
642
643 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
644                                                           AVCodecContext *avctx)
645 {
646     CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
647                                                                    0,
648                                                                    &kCFTypeDictionaryKeyCallBacks,
649                                                                    &kCFTypeDictionaryValueCallBacks);
650
651     CFDictionarySetValue(config_info,
652                          kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
653                          kCFBooleanTrue);
654
655     if (avctx->extradata_size) {
656         CFMutableDictionaryRef avc_info;
657         CFDataRef data = NULL;
658
659         avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
660                                              1,
661                                              &kCFTypeDictionaryKeyCallBacks,
662                                              &kCFTypeDictionaryValueCallBacks);
663
664         switch (codec_type) {
665         case kCMVideoCodecType_MPEG4Video :
666             data = videotoolbox_esds_extradata_create(avctx);
667             if (data)
668                 CFDictionarySetValue(avc_info, CFSTR("esds"), data);
669             break;
670         case kCMVideoCodecType_H264 :
671             data = ff_videotoolbox_avcc_extradata_create(avctx);
672             if (data)
673                 CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
674             break;
675         case kCMVideoCodecType_HEVC :
676             data = ff_videotoolbox_hvcc_extradata_create(avctx);
677             if (data)
678                 CFDictionarySetValue(avc_info, CFSTR("hvcC"), data);
679             break;
680         default:
681             break;
682         }
683
684         CFDictionarySetValue(config_info,
685                 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
686                 avc_info);
687
688         if (data)
689             CFRelease(data);
690
691         CFRelease(avc_info);
692     }
693     return config_info;
694 }
695
696 static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
697                                                              int height,
698                                                              OSType pix_fmt)
699 {
700     CFMutableDictionaryRef buffer_attributes;
701     CFMutableDictionaryRef io_surface_properties;
702     CFNumberRef cv_pix_fmt;
703     CFNumberRef w;
704     CFNumberRef h;
705
706     w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
707     h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
708     cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
709
710     buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
711                                                   4,
712                                                   &kCFTypeDictionaryKeyCallBacks,
713                                                   &kCFTypeDictionaryValueCallBacks);
714     io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
715                                                       0,
716                                                       &kCFTypeDictionaryKeyCallBacks,
717                                                       &kCFTypeDictionaryValueCallBacks);
718
719     if (pix_fmt)
720         CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
721     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
722     CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
723     CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
724 #if TARGET_OS_IPHONE
725     CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
726 #else
727     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
728 #endif
729
730     CFRelease(io_surface_properties);
731     CFRelease(cv_pix_fmt);
732     CFRelease(w);
733     CFRelease(h);
734
735     return buffer_attributes;
736 }
737
738 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
739                                                                    CFDictionaryRef decoder_spec,
740                                                                    int width,
741                                                                    int height)
742 {
743     CMFormatDescriptionRef cm_fmt_desc;
744     OSStatus status;
745
746     status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
747                                             codec_type,
748                                             width,
749                                             height,
750                                             decoder_spec, // Dictionary of extension
751                                             &cm_fmt_desc);
752
753     if (status)
754         return NULL;
755
756     return cm_fmt_desc;
757 }
758
759 static int videotoolbox_default_init(AVCodecContext *avctx)
760 {
761     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
762     OSStatus status;
763     VTDecompressionOutputCallbackRecord decoder_cb;
764     CFDictionaryRef decoder_spec;
765     CFDictionaryRef buf_attr;
766
767     if (!videotoolbox) {
768         av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
769         return -1;
770     }
771
772     switch( avctx->codec_id ) {
773     case AV_CODEC_ID_H263 :
774         videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
775         break;
776     case AV_CODEC_ID_H264 :
777         videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
778         break;
779     case AV_CODEC_ID_HEVC :
780         videotoolbox->cm_codec_type = kCMVideoCodecType_HEVC;
781         break;
782     case AV_CODEC_ID_MPEG1VIDEO :
783         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
784         break;
785     case AV_CODEC_ID_MPEG2VIDEO :
786         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
787         break;
788     case AV_CODEC_ID_MPEG4 :
789         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
790         break;
791     default :
792         break;
793     }
794
795     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
796
797     videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
798                                                                 decoder_spec,
799                                                                 avctx->width,
800                                                                 avctx->height);
801     if (!videotoolbox->cm_fmt_desc) {
802         if (decoder_spec)
803             CFRelease(decoder_spec);
804
805         av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
806         return -1;
807     }
808
809     buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
810                                                      avctx->height,
811                                                      videotoolbox->cv_pix_fmt_type);
812
813     decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
814     decoder_cb.decompressionOutputRefCon   = avctx;
815
816     status = VTDecompressionSessionCreate(NULL,                      // allocator
817                                           videotoolbox->cm_fmt_desc, // videoFormatDescription
818                                           decoder_spec,              // videoDecoderSpecification
819                                           buf_attr,                  // destinationImageBufferAttributes
820                                           &decoder_cb,               // outputCallback
821                                           &videotoolbox->session);   // decompressionSessionOut
822
823     if (decoder_spec)
824         CFRelease(decoder_spec);
825     if (buf_attr)
826         CFRelease(buf_attr);
827
828     switch (status) {
829     case kVTVideoDecoderNotAvailableNowErr:
830         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox session not available.\n");
831         return AVERROR(ENOSYS);
832     case kVTVideoDecoderUnsupportedDataFormatErr:
833         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox does not support this format.\n");
834         return AVERROR(ENOSYS);
835     case kVTVideoDecoderMalfunctionErr:
836         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n");
837         return AVERROR(EINVAL);
838     case kVTVideoDecoderBadDataErr :
839         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n");
840         return AVERROR_INVALIDDATA;
841     case 0:
842         return 0;
843     default:
844         av_log(avctx, AV_LOG_VERBOSE, "Unknown VideoToolbox session creation error %u\n", (unsigned)status);
845         return AVERROR_UNKNOWN;
846     }
847 }
848
849 static void videotoolbox_default_free(AVCodecContext *avctx)
850 {
851     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
852     if (!videotoolbox)
853         return;
854
855     if (videotoolbox->cm_fmt_desc)
856         CFRelease(videotoolbox->cm_fmt_desc);
857
858     if (videotoolbox->session) {
859         VTDecompressionSessionInvalidate(videotoolbox->session);
860         CFRelease(videotoolbox->session);
861     }
862 }
863
864 static int videotoolbox_uninit(AVCodecContext *avctx)
865 {
866     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
867     if (!vtctx)
868         return 0;
869
870     ff_videotoolbox_uninit(avctx);
871
872     if (vtctx->vt_ctx)
873         videotoolbox_default_free(avctx);
874
875     av_buffer_unref(&vtctx->cached_hw_frames_ctx);
876     av_freep(&vtctx->vt_ctx);
877
878     return 0;
879 }
880
881 static int videotoolbox_common_init(AVCodecContext *avctx)
882 {
883     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
884     AVHWFramesContext *hw_frames;
885     int err;
886
887     // Old API - do nothing.
888     if (avctx->hwaccel_context)
889         return 0;
890
891     if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
892         av_log(avctx, AV_LOG_ERROR,
893                "Either hw_frames_ctx or hw_device_ctx must be set.\n");
894         return AVERROR(EINVAL);
895     }
896
897     vtctx->vt_ctx = av_videotoolbox_alloc_context();
898     if (!vtctx->vt_ctx) {
899         err = AVERROR(ENOMEM);
900         goto fail;
901     }
902
903     if (avctx->hw_frames_ctx) {
904         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
905     } else {
906         avctx->hw_frames_ctx = av_hwframe_ctx_alloc(avctx->hw_device_ctx);
907         if (!avctx->hw_frames_ctx) {
908             err = AVERROR(ENOMEM);
909             goto fail;
910         }
911
912         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
913         hw_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
914         hw_frames->sw_format = AV_PIX_FMT_NV12; // same as av_videotoolbox_alloc_context()
915         hw_frames->width = avctx->width;
916         hw_frames->height = avctx->height;
917
918         err = av_hwframe_ctx_init(avctx->hw_frames_ctx);
919         if (err < 0) {
920             av_buffer_unref(&avctx->hw_frames_ctx);
921             goto fail;
922         }
923     }
924
925     vtctx->cached_hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
926     if (!vtctx->cached_hw_frames_ctx) {
927         err = AVERROR(ENOMEM);
928         goto fail;
929     }
930
931     vtctx->vt_ctx->cv_pix_fmt_type =
932         av_map_videotoolbox_format_from_pixfmt(hw_frames->sw_format);
933     if (!vtctx->vt_ctx->cv_pix_fmt_type) {
934         av_log(avctx, AV_LOG_ERROR, "Unknown sw_format.\n");
935         err = AVERROR(EINVAL);
936         goto fail;
937     }
938
939     err = videotoolbox_default_init(avctx);
940     if (err < 0)
941         goto fail;
942
943     return 0;
944
945 fail:
946     videotoolbox_uninit(avctx);
947     return err;
948 }
949
950 static int videotoolbox_frame_params(AVCodecContext *avctx,
951                                      AVBufferRef *hw_frames_ctx)
952 {
953     AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
954
955     frames_ctx->format            = AV_PIX_FMT_VIDEOTOOLBOX;
956     frames_ctx->width             = avctx->coded_width;
957     frames_ctx->height            = avctx->coded_height;
958     frames_ctx->sw_format         = AV_PIX_FMT_NV12;
959
960     return 0;
961 }
962
963 AVHWAccel ff_h263_videotoolbox_hwaccel = {
964     .name           = "h263_videotoolbox",
965     .type           = AVMEDIA_TYPE_VIDEO,
966     .id             = AV_CODEC_ID_H263,
967     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
968     .alloc_frame    = ff_videotoolbox_alloc_frame,
969     .start_frame    = videotoolbox_mpeg_start_frame,
970     .decode_slice   = videotoolbox_mpeg_decode_slice,
971     .end_frame      = videotoolbox_mpeg_end_frame,
972     .frame_params   = videotoolbox_frame_params,
973     .init           = videotoolbox_common_init,
974     .uninit         = videotoolbox_uninit,
975     .priv_data_size = sizeof(VTContext),
976 };
977
978 AVHWAccel ff_hevc_videotoolbox_hwaccel = {
979     .name           = "hevc_videotoolbox",
980     .type           = AVMEDIA_TYPE_VIDEO,
981     .id             = AV_CODEC_ID_HEVC,
982     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
983     .alloc_frame    = ff_videotoolbox_alloc_frame,
984     .start_frame    = ff_videotoolbox_h264_start_frame,
985     .decode_slice   = ff_videotoolbox_h264_decode_slice,
986     .end_frame      = videotoolbox_hevc_end_frame,
987     .frame_params   = videotoolbox_frame_params,
988     .init           = videotoolbox_common_init,
989     .uninit         = ff_videotoolbox_uninit,
990     .priv_data_size = sizeof(VTContext),
991 };
992
993 AVHWAccel ff_h264_videotoolbox_hwaccel = {
994     .name           = "h264_videotoolbox",
995     .type           = AVMEDIA_TYPE_VIDEO,
996     .id             = AV_CODEC_ID_H264,
997     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
998     .alloc_frame    = ff_videotoolbox_alloc_frame,
999     .start_frame    = ff_videotoolbox_h264_start_frame,
1000     .decode_slice   = ff_videotoolbox_h264_decode_slice,
1001     .end_frame      = videotoolbox_h264_end_frame,
1002     .frame_params   = videotoolbox_frame_params,
1003     .init           = videotoolbox_common_init,
1004     .uninit         = videotoolbox_uninit,
1005     .priv_data_size = sizeof(VTContext),
1006 };
1007
1008 AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
1009     .name           = "mpeg1_videotoolbox",
1010     .type           = AVMEDIA_TYPE_VIDEO,
1011     .id             = AV_CODEC_ID_MPEG1VIDEO,
1012     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1013     .alloc_frame    = ff_videotoolbox_alloc_frame,
1014     .start_frame    = videotoolbox_mpeg_start_frame,
1015     .decode_slice   = videotoolbox_mpeg_decode_slice,
1016     .end_frame      = videotoolbox_mpeg_end_frame,
1017     .frame_params   = videotoolbox_frame_params,
1018     .init           = videotoolbox_common_init,
1019     .uninit         = videotoolbox_uninit,
1020     .priv_data_size = sizeof(VTContext),
1021 };
1022
1023 AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
1024     .name           = "mpeg2_videotoolbox",
1025     .type           = AVMEDIA_TYPE_VIDEO,
1026     .id             = AV_CODEC_ID_MPEG2VIDEO,
1027     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1028     .alloc_frame    = ff_videotoolbox_alloc_frame,
1029     .start_frame    = videotoolbox_mpeg_start_frame,
1030     .decode_slice   = videotoolbox_mpeg_decode_slice,
1031     .end_frame      = videotoolbox_mpeg_end_frame,
1032     .frame_params   = videotoolbox_frame_params,
1033     .init           = videotoolbox_common_init,
1034     .uninit         = videotoolbox_uninit,
1035     .priv_data_size = sizeof(VTContext),
1036 };
1037
1038 AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
1039     .name           = "mpeg4_videotoolbox",
1040     .type           = AVMEDIA_TYPE_VIDEO,
1041     .id             = AV_CODEC_ID_MPEG4,
1042     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1043     .alloc_frame    = ff_videotoolbox_alloc_frame,
1044     .start_frame    = videotoolbox_mpeg_start_frame,
1045     .decode_slice   = videotoolbox_mpeg_decode_slice,
1046     .end_frame      = videotoolbox_mpeg_end_frame,
1047     .frame_params   = videotoolbox_frame_params,
1048     .init           = videotoolbox_common_init,
1049     .uninit         = videotoolbox_uninit,
1050     .priv_data_size = sizeof(VTContext),
1051 };
1052
1053 AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
1054 {
1055     AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
1056
1057     if (ret) {
1058         ret->output_callback = videotoolbox_decoder_callback;
1059         ret->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1060     }
1061
1062     return ret;
1063 }
1064
1065 int av_videotoolbox_default_init(AVCodecContext *avctx)
1066 {
1067     return av_videotoolbox_default_init2(avctx, NULL);
1068 }
1069
1070 int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
1071 {
1072     avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context();
1073     if (!avctx->hwaccel_context)
1074         return AVERROR(ENOMEM);
1075     return videotoolbox_default_init(avctx);
1076 }
1077
1078 void av_videotoolbox_default_free(AVCodecContext *avctx)
1079 {
1080
1081     videotoolbox_default_free(avctx);
1082     av_freep(&avctx->hwaccel_context);
1083 }
1084 #endif /* CONFIG_VIDEOTOOLBOX */