lavc: VAAPI H.264 encoder
[ffmpeg.git] / libavcodec / vaapi_encode_h264.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include <va/va.h>
20 #include <va/va_enc_h264.h>
21
22 #include "libavutil/avassert.h"
23 #include "libavutil/internal.h"
24 #include "libavutil/opt.h"
25 #include "libavutil/pixfmt.h"
26
27 #include "avcodec.h"
28 #include "h264.h"
29 #include "internal.h"
30 #include "vaapi_encode.h"
31 #include "vaapi_encode_h26x.h"
32
33 enum {
34     SLICE_TYPE_P  = 0,
35     SLICE_TYPE_B  = 1,
36     SLICE_TYPE_I  = 2,
37     SLICE_TYPE_SP = 3,
38     SLICE_TYPE_SI = 4,
39 };
40
41 // This structure contains all possibly-useful per-sequence syntax elements
42 // which are not already contained in the various VAAPI structures.
43 typedef struct VAAPIEncodeH264MiscSequenceParams {
44     unsigned int profile_idc;
45     char constraint_set0_flag;
46     char constraint_set1_flag;
47     char constraint_set2_flag;
48     char constraint_set3_flag;
49     char constraint_set4_flag;
50     char constraint_set5_flag;
51
52     char separate_colour_plane_flag;
53     char qpprime_y_zero_transform_bypass_flag;
54
55     char gaps_in_frame_num_allowed_flag;
56     char delta_pic_order_always_zero_flag;
57     char bottom_field_pic_order_in_frame_present_flag;
58
59     unsigned int num_slice_groups_minus1;
60     unsigned int slice_group_map_type;
61
62     int pic_init_qs_minus26;
63
64     char vui_parameters_present_flag;
65 } VAAPIEncodeH264MiscSequenceParams;
66
67 // This structure contains all possibly-useful per-slice syntax elements
68 // which are not already contained in the various VAAPI structures.
69 typedef struct VAAPIEncodeH264MiscSliceParams {
70     unsigned int nal_unit_type;
71     unsigned int nal_ref_idc;
72
73     unsigned int colour_plane_id;
74     char field_pic_flag;
75     char bottom_field_flag;
76
77     unsigned int redundant_pic_cnt;
78
79     char sp_for_switch_flag;
80     int slice_qs_delta;
81
82     char ref_pic_list_modification_flag_l0;
83     char ref_pic_list_modification_flag_l1;
84
85     char no_output_of_prior_pics_flag;
86     char long_term_reference_flag;
87     char adaptive_ref_pic_marking_mode_flag;
88 } VAAPIEncodeH264MiscSliceParams;
89
90 typedef struct VAAPIEncodeH264Slice {
91     VAAPIEncodeH264MiscSliceParams misc_slice_params;
92 } VAAPIEncodeH264Slice;
93
94 typedef struct VAAPIEncodeH264Context {
95     VAAPIEncodeH264MiscSequenceParams misc_sequence_params;
96
97     int mb_width;
98     int mb_height;
99
100     int fixed_qp_idr;
101     int fixed_qp_p;
102     int fixed_qp_b;
103
104     int64_t idr_pic_count;
105     int64_t last_idr_frame;
106
107     // RefPicList management.
108 } VAAPIEncodeH264Context;
109
110
111 #define vseq_var(name)     vseq->name, name
112 #define vseq_field(name)   vseq->seq_fields.bits.name, name
113 #define vpic_var(name)     vpic->name, name
114 #define vpic_field(name)   vpic->pic_fields.bits.name, name
115 #define vslice_var(name)   vslice->name, name
116 #define vslice_field(name) vslice->slice_fields.bits.name, name
117 #define mseq_var(name)     mseq->name, name
118 #define mslice_var(name)   mslice->name, name
119
120 static void vaapi_encode_h264_write_nal_header(PutBitContext *pbc,
121                                                int nal_unit_type, int nal_ref_idc)
122 {
123     u(1, 0, forbidden_zero_bit);
124     u(2, nal_ref_idc, nal_ref_idc);
125     u(5, nal_unit_type, nal_unit_type);
126 }
127
128 static void vaapi_encode_h264_write_trailing_rbsp(PutBitContext *pbc)
129 {
130     u(1, 1, rbsp_stop_one_bit);
131     while (put_bits_count(pbc) & 7)
132         u(1, 0, rbsp_alignment_zero_bit);
133 }
134
135 static void vaapi_encode_h264_write_sps(PutBitContext *pbc,
136                                         VAAPIEncodeContext *ctx)
137 {
138     VAEncSequenceParameterBufferH264  *vseq = ctx->codec_sequence_params;
139     VAAPIEncodeH264Context            *priv = ctx->priv_data;
140     VAAPIEncodeH264MiscSequenceParams *mseq = &priv->misc_sequence_params;
141     int i;
142
143     vaapi_encode_h264_write_nal_header(pbc, NAL_SPS, 3);
144
145     u(8, mseq_var(profile_idc));
146     u(1, mseq_var(constraint_set0_flag));
147     u(1, mseq_var(constraint_set1_flag));
148     u(1, mseq_var(constraint_set2_flag));
149     u(1, mseq_var(constraint_set3_flag));
150     u(1, mseq_var(constraint_set4_flag));
151     u(1, mseq_var(constraint_set5_flag));
152     u(2, 0, reserved_zero_2bits);
153
154     u(8, vseq_var(level_idc));
155
156     ue(vseq_var(seq_parameter_set_id));
157
158     if (mseq->profile_idc == 100 || mseq->profile_idc == 110 ||
159         mseq->profile_idc == 122 || mseq->profile_idc == 244 ||
160         mseq->profile_idc ==  44 || mseq->profile_idc ==  83 ||
161         mseq->profile_idc ==  86 || mseq->profile_idc == 118 ||
162         mseq->profile_idc == 128 || mseq->profile_idc == 138) {
163         ue(vseq_field(chroma_format_idc));
164
165         if (vseq->seq_fields.bits.chroma_format_idc == 3)
166             u(1, mseq_var(separate_colour_plane_flag));
167
168         ue(vseq_var(bit_depth_luma_minus8));
169         ue(vseq_var(bit_depth_chroma_minus8));
170
171         u(1, mseq_var(qpprime_y_zero_transform_bypass_flag));
172
173         u(1, vseq_field(seq_scaling_matrix_present_flag));
174         if (vseq->seq_fields.bits.seq_scaling_matrix_present_flag) {
175             av_assert0(0 && "scaling matrices not supported");
176         }
177     }
178
179     ue(vseq_field(log2_max_frame_num_minus4));
180     ue(vseq_field(pic_order_cnt_type));
181
182     if (vseq->seq_fields.bits.pic_order_cnt_type == 0) {
183         ue(vseq_field(log2_max_pic_order_cnt_lsb_minus4));
184     } else if (vseq->seq_fields.bits.pic_order_cnt_type == 1) {
185         u(1, mseq_var(delta_pic_order_always_zero_flag));
186         se(vseq_var(offset_for_non_ref_pic));
187         se(vseq_var(offset_for_top_to_bottom_field));
188         ue(vseq_var(num_ref_frames_in_pic_order_cnt_cycle));
189
190         for (i = 0; i < vseq->num_ref_frames_in_pic_order_cnt_cycle; i++)
191             se(vseq_var(offset_for_ref_frame[i]));
192     }
193
194     ue(vseq_var(max_num_ref_frames));
195     u(1, mseq_var(gaps_in_frame_num_allowed_flag));
196
197     ue(vseq->picture_width_in_mbs  - 1, pic_width_in_mbs_minus1);
198     ue(vseq->picture_height_in_mbs - 1, pic_height_in_mbs_minus1);
199
200     u(1, vseq_field(frame_mbs_only_flag));
201     if (!vseq->seq_fields.bits.frame_mbs_only_flag)
202         u(1, vseq_field(mb_adaptive_frame_field_flag));
203
204     u(1, vseq_field(direct_8x8_inference_flag));
205
206     u(1, vseq_var(frame_cropping_flag));
207     if (vseq->frame_cropping_flag) {
208         ue(vseq_var(frame_crop_left_offset));
209         ue(vseq_var(frame_crop_right_offset));
210         ue(vseq_var(frame_crop_top_offset));
211         ue(vseq_var(frame_crop_bottom_offset));
212     }
213
214     u(1, mseq_var(vui_parameters_present_flag));
215
216     vaapi_encode_h264_write_trailing_rbsp(pbc);
217 }
218
219 static void vaapi_encode_h264_write_pps(PutBitContext *pbc,
220                                         VAAPIEncodeContext *ctx)
221 {
222     VAEncPictureParameterBufferH264   *vpic = ctx->codec_picture_params;
223     VAAPIEncodeH264Context            *priv = ctx->priv_data;
224     VAAPIEncodeH264MiscSequenceParams *mseq = &priv->misc_sequence_params;
225
226     vaapi_encode_h264_write_nal_header(pbc, NAL_PPS, 3);
227
228     ue(vpic_var(pic_parameter_set_id));
229     ue(vpic_var(seq_parameter_set_id));
230
231     u(1, vpic_field(entropy_coding_mode_flag));
232     u(1, mseq_var(bottom_field_pic_order_in_frame_present_flag));
233
234     ue(mseq_var(num_slice_groups_minus1));
235     if (mseq->num_slice_groups_minus1 > 0) {
236         ue(mseq_var(slice_group_map_type));
237         av_assert0(0 && "slice groups not supported");
238     }
239
240     ue(vpic_var(num_ref_idx_l0_active_minus1));
241     ue(vpic_var(num_ref_idx_l1_active_minus1));
242
243     u(1, vpic_field(weighted_pred_flag));
244     u(2, vpic_field(weighted_bipred_idc));
245
246     se(vpic->pic_init_qp - 26, pic_init_qp_minus26);
247     se(mseq_var(pic_init_qs_minus26));
248     se(vpic_var(chroma_qp_index_offset));
249
250     u(1, vpic_field(deblocking_filter_control_present_flag));
251     u(1, vpic_field(constrained_intra_pred_flag));
252     u(1, vpic_field(redundant_pic_cnt_present_flag));
253     u(1, vpic_field(transform_8x8_mode_flag));
254
255     u(1, vpic_field(pic_scaling_matrix_present_flag));
256     if (vpic->pic_fields.bits.pic_scaling_matrix_present_flag) {
257         av_assert0(0 && "scaling matrices not supported");
258     }
259
260     se(vpic_var(second_chroma_qp_index_offset));
261
262     vaapi_encode_h264_write_trailing_rbsp(pbc);
263 }
264
265 static void vaapi_encode_h264_write_slice_header2(PutBitContext *pbc,
266                                                   VAAPIEncodeContext *ctx,
267                                                   VAAPIEncodePicture *pic,
268                                                   VAAPIEncodeSlice *slice)
269 {
270     VAEncSequenceParameterBufferH264  *vseq = ctx->codec_sequence_params;
271     VAEncPictureParameterBufferH264   *vpic = pic->codec_picture_params;
272     VAEncSliceParameterBufferH264   *vslice = slice->codec_slice_params;
273     VAAPIEncodeH264Context            *priv = ctx->priv_data;
274     VAAPIEncodeH264MiscSequenceParams *mseq = &priv->misc_sequence_params;
275     VAAPIEncodeH264Slice            *pslice = slice->priv_data;
276     VAAPIEncodeH264MiscSliceParams  *mslice = &pslice->misc_slice_params;
277
278     vaapi_encode_h264_write_nal_header(pbc, mslice->nal_unit_type,
279                                        mslice->nal_ref_idc);
280
281     ue(vslice->macroblock_address, first_mb_in_slice);
282     ue(vslice_var(slice_type));
283     ue(vpic_var(pic_parameter_set_id));
284
285     if (mseq->separate_colour_plane_flag) {
286         u(2, mslice_var(colour_plane_id));
287     }
288
289     u(4 + vseq->seq_fields.bits.log2_max_frame_num_minus4,
290       (vpic->frame_num &
291        ((1 << (4 + vseq->seq_fields.bits.log2_max_frame_num_minus4)) - 1)),
292       frame_num);
293
294     if (!vseq->seq_fields.bits.frame_mbs_only_flag) {
295         u(1, mslice_var(field_pic_flag));
296         if (mslice->field_pic_flag)
297             u(1, mslice_var(bottom_field_flag));
298     }
299
300     if (vpic->pic_fields.bits.idr_pic_flag) {
301         ue(vslice_var(idr_pic_id));
302     }
303
304     if (vseq->seq_fields.bits.pic_order_cnt_type == 0) {
305         u(4 + vseq->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4,
306           vslice_var(pic_order_cnt_lsb));
307         if (mseq->bottom_field_pic_order_in_frame_present_flag &&
308             !mslice->field_pic_flag) {
309             se(vslice_var(delta_pic_order_cnt_bottom));
310         }
311     }
312
313     if (vseq->seq_fields.bits.pic_order_cnt_type == 1 &&
314         !vseq->seq_fields.bits.delta_pic_order_always_zero_flag) {
315         se(vslice_var(delta_pic_order_cnt[0]));
316         if (mseq->bottom_field_pic_order_in_frame_present_flag &&
317             !mslice->field_pic_flag) {
318             se(vslice_var(delta_pic_order_cnt[1]));
319         }
320     }
321
322     if (vpic->pic_fields.bits.redundant_pic_cnt_present_flag) {
323         ue(mslice_var(redundant_pic_cnt));
324     }
325
326     if (vslice->slice_type == SLICE_TYPE_B) {
327         u(1, vslice_var(direct_spatial_mv_pred_flag));
328     }
329
330     if (vslice->slice_type == SLICE_TYPE_P ||
331         vslice->slice_type == SLICE_TYPE_SP ||
332         vslice->slice_type == SLICE_TYPE_B) {
333         u(1, vslice_var(num_ref_idx_active_override_flag));
334         if (vslice->num_ref_idx_active_override_flag) {
335             ue(vslice_var(num_ref_idx_l0_active_minus1));
336             if (vslice->slice_type == SLICE_TYPE_B)
337                 ue(vslice_var(num_ref_idx_l1_active_minus1));
338         }
339     }
340
341     if (mslice->nal_unit_type == 20 || mslice->nal_unit_type == 21) {
342         av_assert0(0 && "no MVC support");
343     } else {
344         if (vslice->slice_type % 5 != 2 && vslice->slice_type % 5 != 4) {
345             u(1, mslice_var(ref_pic_list_modification_flag_l0));
346             if (mslice->ref_pic_list_modification_flag_l0) {
347                 av_assert0(0 && "ref pic list modification");
348             }
349         }
350         if (vslice->slice_type % 5 == 1) {
351             u(1, mslice_var(ref_pic_list_modification_flag_l1));
352             if (mslice->ref_pic_list_modification_flag_l1) {
353                 av_assert0(0 && "ref pic list modification");
354             }
355         }
356     }
357
358     if ((vpic->pic_fields.bits.weighted_pred_flag &&
359          (vslice->slice_type == SLICE_TYPE_P ||
360           vslice->slice_type == SLICE_TYPE_SP)) ||
361         (vpic->pic_fields.bits.weighted_bipred_idc == 1 &&
362          vslice->slice_type == SLICE_TYPE_B)) {
363         av_assert0(0 && "prediction weights not supported");
364     }
365
366     av_assert0(mslice->nal_ref_idc > 0 ==
367                vpic->pic_fields.bits.reference_pic_flag);
368     if (mslice->nal_ref_idc != 0) {
369         if (vpic->pic_fields.bits.idr_pic_flag) {
370             u(1, mslice_var(no_output_of_prior_pics_flag));
371             u(1, mslice_var(long_term_reference_flag));
372         } else {
373             u(1, mslice_var(adaptive_ref_pic_marking_mode_flag));
374             if (mslice->adaptive_ref_pic_marking_mode_flag) {
375                 av_assert0(0 && "MMCOs not supported");
376             }
377         }
378     }
379
380     if (vpic->pic_fields.bits.entropy_coding_mode_flag &&
381         vslice->slice_type != SLICE_TYPE_I &&
382         vslice->slice_type != SLICE_TYPE_SI) {
383         ue(vslice_var(cabac_init_idc));
384     }
385
386     se(vslice_var(slice_qp_delta));
387     if (vslice->slice_type == SLICE_TYPE_SP ||
388         vslice->slice_type == SLICE_TYPE_SI) {
389         if (vslice->slice_type == SLICE_TYPE_SP)
390             u(1, mslice_var(sp_for_switch_flag));
391         se(mslice_var(slice_qs_delta));
392     }
393
394     if (vpic->pic_fields.bits.deblocking_filter_control_present_flag) {
395         ue(vslice_var(disable_deblocking_filter_idc));
396         if (vslice->disable_deblocking_filter_idc != 1) {
397             se(vslice_var(slice_alpha_c0_offset_div2));
398             se(vslice_var(slice_beta_offset_div2));
399         }
400     }
401
402     if (mseq->num_slice_groups_minus1 > 0 &&
403         mseq->slice_group_map_type >= 3 && mseq->slice_group_map_type <= 5) {
404         av_assert0(0 && "slice groups not supported");
405     }
406
407     // No alignment - this need not be a byte boundary.
408 }
409
410 static int vaapi_encode_h264_write_sequence_header(AVCodecContext *avctx,
411                                                    char *data, size_t *data_len)
412 {
413     VAAPIEncodeContext *ctx = avctx->priv_data;
414     PutBitContext pbc;
415     char tmp[256];
416     int err;
417     size_t nal_len, bit_len, bit_pos, next_len;
418
419     bit_len = *data_len;
420     bit_pos = 0;
421
422     init_put_bits(&pbc, tmp, sizeof(tmp));
423     vaapi_encode_h264_write_sps(&pbc, ctx);
424     nal_len = put_bits_count(&pbc);
425     flush_put_bits(&pbc);
426
427     next_len = bit_len - bit_pos;
428     err = ff_vaapi_encode_h26x_nal_unit_to_byte_stream(data + bit_pos / 8,
429                                                        &next_len,
430                                                        tmp, nal_len);
431     if (err < 0)
432         return err;
433     bit_pos += next_len;
434
435     init_put_bits(&pbc, tmp, sizeof(tmp));
436     vaapi_encode_h264_write_pps(&pbc, ctx);
437     nal_len = put_bits_count(&pbc);
438     flush_put_bits(&pbc);
439
440     next_len = bit_len - bit_pos;
441     err = ff_vaapi_encode_h26x_nal_unit_to_byte_stream(data + bit_pos / 8,
442                                                        &next_len,
443                                                        tmp, nal_len);
444     if (err < 0)
445         return err;
446     bit_pos += next_len;
447
448     *data_len = bit_pos;
449     return 0;
450 }
451
452 static int vaapi_encode_h264_write_slice_header(AVCodecContext *avctx,
453                                                 VAAPIEncodePicture *pic,
454                                                 VAAPIEncodeSlice *slice,
455                                                 char *data, size_t *data_len)
456 {
457     VAAPIEncodeContext *ctx = avctx->priv_data;
458     PutBitContext pbc;
459     char tmp[256];
460     size_t header_len;
461
462     init_put_bits(&pbc, tmp, sizeof(tmp));
463     vaapi_encode_h264_write_slice_header2(&pbc, ctx, pic, slice);
464     header_len = put_bits_count(&pbc);
465     flush_put_bits(&pbc);
466
467     return ff_vaapi_encode_h26x_nal_unit_to_byte_stream(data, data_len,
468                                                         tmp, header_len);
469 }
470
471 static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
472 {
473     VAAPIEncodeContext                 *ctx = avctx->priv_data;
474     VAEncSequenceParameterBufferH264  *vseq = ctx->codec_sequence_params;
475     VAEncPictureParameterBufferH264   *vpic = ctx->codec_picture_params;
476     VAAPIEncodeH264Context            *priv = ctx->priv_data;
477     VAAPIEncodeH264MiscSequenceParams *mseq = &priv->misc_sequence_params;
478     int i;
479
480     {
481         vseq->seq_parameter_set_id = 0;
482
483         vseq->level_idc = avctx->level;
484
485         vseq->max_num_ref_frames = 2;
486
487         vseq->picture_width_in_mbs  = priv->mb_width;
488         vseq->picture_height_in_mbs = priv->mb_height;
489
490         vseq->seq_fields.bits.chroma_format_idc = 1;
491         vseq->seq_fields.bits.frame_mbs_only_flag = 1;
492         vseq->seq_fields.bits.direct_8x8_inference_flag = 1;
493         vseq->seq_fields.bits.log2_max_frame_num_minus4 = 4;
494         vseq->seq_fields.bits.pic_order_cnt_type = 0;
495
496         if (ctx->input_width  != ctx->aligned_width ||
497             ctx->input_height != ctx->aligned_height) {
498             vseq->frame_cropping_flag = 1;
499
500             vseq->frame_crop_left_offset   = 0;
501             vseq->frame_crop_right_offset  =
502                 (ctx->aligned_width - ctx->input_width) / 2;
503             vseq->frame_crop_top_offset    = 0;
504             vseq->frame_crop_bottom_offset =
505                 (ctx->aligned_height - ctx->input_height) / 2;
506         } else {
507             vseq->frame_cropping_flag = 0;
508         }
509     }
510
511     {
512         vpic->CurrPic.picture_id = VA_INVALID_ID;
513         vpic->CurrPic.flags      = VA_PICTURE_H264_INVALID;
514
515         for (i = 0; i < FF_ARRAY_ELEMS(vpic->ReferenceFrames); i++) {
516             vpic->ReferenceFrames[i].picture_id = VA_INVALID_ID;
517             vpic->ReferenceFrames[i].flags      = VA_PICTURE_H264_INVALID;
518         }
519
520         vpic->coded_buf = VA_INVALID_ID;
521
522         vpic->pic_parameter_set_id = 0;
523         vpic->seq_parameter_set_id = 0;
524
525         vpic->num_ref_idx_l0_active_minus1 = 0;
526         vpic->num_ref_idx_l1_active_minus1 = 0;
527
528         vpic->pic_fields.bits.entropy_coding_mode_flag =
529             ((avctx->profile & 0xff) != 66);
530         vpic->pic_fields.bits.weighted_pred_flag = 0;
531         vpic->pic_fields.bits.weighted_bipred_idc = 0;
532         vpic->pic_fields.bits.transform_8x8_mode_flag =
533             ((avctx->profile & 0xff) >= 100);
534
535         vpic->pic_init_qp = priv->fixed_qp_idr;
536     }
537
538     {
539         mseq->profile_idc = avctx->profile & 0xff;
540
541         if (avctx->profile & FF_PROFILE_H264_CONSTRAINED)
542             mseq->constraint_set1_flag = 1;
543         if (avctx->profile & FF_PROFILE_H264_INTRA)
544             mseq->constraint_set3_flag = 1;
545     }
546
547     return 0;
548 }
549
550 static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
551                                                  VAAPIEncodePicture *pic)
552 {
553     VAAPIEncodeContext                *ctx = avctx->priv_data;
554     VAEncSequenceParameterBufferH264 *vseq = ctx->codec_sequence_params;
555     VAEncPictureParameterBufferH264  *vpic = pic->codec_picture_params;
556     VAAPIEncodeH264Context           *priv = ctx->priv_data;
557     int i;
558
559     if (pic->type == PICTURE_TYPE_IDR) {
560         av_assert0(pic->display_order == pic->encode_order);
561         priv->last_idr_frame = pic->display_order;
562     } else {
563         av_assert0(pic->display_order > priv->last_idr_frame);
564     }
565
566     vpic->frame_num = (pic->encode_order - priv->last_idr_frame) &
567         ((1 << (4 + vseq->seq_fields.bits.log2_max_frame_num_minus4)) - 1);
568
569     vpic->CurrPic.picture_id          = pic->recon_surface;
570     vpic->CurrPic.frame_idx           = vpic->frame_num;
571     vpic->CurrPic.flags               = 0;
572     vpic->CurrPic.TopFieldOrderCnt    = pic->display_order;
573     vpic->CurrPic.BottomFieldOrderCnt = pic->display_order;
574
575     for (i = 0; i < pic->nb_refs; i++) {
576         VAAPIEncodePicture *ref = pic->refs[i];
577         av_assert0(ref && ref->encode_order >= priv->last_idr_frame);
578         vpic->ReferenceFrames[i].picture_id = ref->recon_surface;
579         vpic->ReferenceFrames[i].frame_idx =
580             ref->encode_order - priv->last_idr_frame;
581         vpic->ReferenceFrames[i].flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE;
582         vpic->ReferenceFrames[i].TopFieldOrderCnt    = ref->display_order;
583         vpic->ReferenceFrames[i].BottomFieldOrderCnt = ref->display_order;
584     }
585     for (; i < FF_ARRAY_ELEMS(vpic->ReferenceFrames); i++) {
586         vpic->ReferenceFrames[i].picture_id = VA_INVALID_ID;
587         vpic->ReferenceFrames[i].flags = VA_PICTURE_H264_INVALID;
588     }
589
590     vpic->coded_buf = pic->output_buffer;
591
592     vpic->pic_fields.bits.idr_pic_flag = (pic->type == PICTURE_TYPE_IDR);
593     vpic->pic_fields.bits.reference_pic_flag = (pic->type != PICTURE_TYPE_B);
594
595     pic->nb_slices = 1;
596
597     return 0;
598 }
599
600 static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
601                                                VAAPIEncodePicture *pic,
602                                                VAAPIEncodeSlice *slice)
603 {
604     VAAPIEncodeContext                 *ctx = avctx->priv_data;
605     VAEncSequenceParameterBufferH264  *vseq = ctx->codec_sequence_params;
606     VAEncPictureParameterBufferH264   *vpic = pic->codec_picture_params;
607     VAEncSliceParameterBufferH264   *vslice = slice->codec_slice_params;
608     VAAPIEncodeH264Context            *priv = ctx->priv_data;
609     VAAPIEncodeH264Slice            *pslice;
610     VAAPIEncodeH264MiscSliceParams  *mslice;
611     int i;
612
613     slice->priv_data = av_mallocz(sizeof(*pslice));
614     if (!slice->priv_data)
615         return AVERROR(ENOMEM);
616     pslice = slice->priv_data;
617     mslice = &pslice->misc_slice_params;
618
619     if (pic->type == PICTURE_TYPE_IDR)
620         mslice->nal_unit_type = NAL_IDR_SLICE;
621     else
622         mslice->nal_unit_type = NAL_SLICE;
623
624     switch (pic->type) {
625     case PICTURE_TYPE_IDR:
626         vslice->slice_type  = SLICE_TYPE_I;
627         mslice->nal_ref_idc = 3;
628         break;
629     case PICTURE_TYPE_I:
630         vslice->slice_type  = SLICE_TYPE_I;
631         mslice->nal_ref_idc = 2;
632         break;
633     case PICTURE_TYPE_P:
634         vslice->slice_type  = SLICE_TYPE_P;
635         mslice->nal_ref_idc = 1;
636         break;
637     case PICTURE_TYPE_B:
638         vslice->slice_type  = SLICE_TYPE_B;
639         mslice->nal_ref_idc = 0;
640         break;
641     default:
642         av_assert0(0 && "invalid picture type");
643     }
644
645     // Only one slice per frame.
646     vslice->macroblock_address = 0;
647     vslice->num_macroblocks = priv->mb_width * priv->mb_height;
648
649     vslice->macroblock_info = VA_INVALID_ID;
650
651     vslice->pic_parameter_set_id = vpic->pic_parameter_set_id;
652     vslice->idr_pic_id = priv->idr_pic_count++;
653
654     vslice->pic_order_cnt_lsb = pic->display_order &
655         ((1 << (4 + vseq->seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4)) - 1);
656
657     for (i = 0; i < FF_ARRAY_ELEMS(vslice->RefPicList0); i++) {
658         vslice->RefPicList0[i].picture_id = VA_INVALID_ID;
659         vslice->RefPicList0[i].flags      = VA_PICTURE_H264_INVALID;
660         vslice->RefPicList1[i].picture_id = VA_INVALID_ID;
661         vslice->RefPicList1[i].flags      = VA_PICTURE_H264_INVALID;
662     }
663
664     av_assert0(pic->nb_refs <= 2);
665     if (pic->nb_refs >= 1) {
666         // Backward reference for P or B frame.
667         av_assert0(pic->type == PICTURE_TYPE_P ||
668                    pic->type == PICTURE_TYPE_B);
669
670         vslice->num_ref_idx_l0_active_minus1 = 0;
671         vslice->RefPicList0[0] = vpic->ReferenceFrames[0];
672     }
673     if (pic->nb_refs >= 2) {
674         // Forward reference for B frame.
675         av_assert0(pic->type == PICTURE_TYPE_B);
676
677         vslice->num_ref_idx_l1_active_minus1 = 0;
678         vslice->RefPicList1[0] = vpic->ReferenceFrames[1];
679     }
680
681     if (pic->type == PICTURE_TYPE_B)
682         vslice->slice_qp_delta = priv->fixed_qp_b - vpic->pic_init_qp;
683     else if (pic->type == PICTURE_TYPE_P)
684         vslice->slice_qp_delta = priv->fixed_qp_p - vpic->pic_init_qp;
685     else
686         vslice->slice_qp_delta = priv->fixed_qp_idr - vpic->pic_init_qp;
687
688     vslice->direct_spatial_mv_pred_flag = 1;
689
690     return 0;
691 }
692
693 static VAConfigAttrib vaapi_encode_h264_config_attributes[] = {
694     { .type  = VAConfigAttribRTFormat,
695       .value = VA_RT_FORMAT_YUV420 },
696     { .type  = VAConfigAttribRateControl,
697       .value = VA_RC_CQP },
698     { .type  = VAConfigAttribEncPackedHeaders,
699       .value = (VA_ENC_PACKED_HEADER_SEQUENCE |
700                 VA_ENC_PACKED_HEADER_SLICE) },
701 };
702
703 static av_cold int vaapi_encode_h264_init_internal(AVCodecContext *avctx)
704 {
705     VAAPIEncodeContext      *ctx = avctx->priv_data;
706     VAAPIEncodeH264Context *priv = ctx->priv_data;
707
708     switch (avctx->profile) {
709     case FF_PROFILE_H264_CONSTRAINED_BASELINE:
710         ctx->va_profile = VAProfileH264ConstrainedBaseline;
711         break;
712     case FF_PROFILE_H264_BASELINE:
713         ctx->va_profile = VAProfileH264Baseline;
714         break;
715     case FF_PROFILE_H264_MAIN:
716         ctx->va_profile = VAProfileH264Main;
717         break;
718     case FF_PROFILE_H264_EXTENDED:
719         av_log(avctx, AV_LOG_ERROR, "H.264 extended profile "
720                "is not supported.\n");
721         return AVERROR_PATCHWELCOME;
722     case FF_PROFILE_UNKNOWN:
723     case FF_PROFILE_H264_HIGH:
724         ctx->va_profile = VAProfileH264High;
725         break;
726     case FF_PROFILE_H264_HIGH_10:
727     case FF_PROFILE_H264_HIGH_10_INTRA:
728         av_log(avctx, AV_LOG_ERROR, "H.264 10-bit profiles "
729                "are not supported.\n");
730         return AVERROR_PATCHWELCOME;
731     case FF_PROFILE_H264_HIGH_422:
732     case FF_PROFILE_H264_HIGH_422_INTRA:
733     case FF_PROFILE_H264_HIGH_444:
734     case FF_PROFILE_H264_HIGH_444_PREDICTIVE:
735     case FF_PROFILE_H264_HIGH_444_INTRA:
736     case FF_PROFILE_H264_CAVLC_444:
737         av_log(avctx, AV_LOG_ERROR, "H.264 non-4:2:0 profiles "
738                "are not supported.\n");
739         return AVERROR_PATCHWELCOME;
740     default:
741         av_log(avctx, AV_LOG_ERROR, "Unknown H.264 profile %d.\n",
742                avctx->profile);
743         return AVERROR(EINVAL);
744     }
745     ctx->va_entrypoint = VAEntrypointEncSlice;
746
747     ctx->va_rc_mode  = VA_RC_CQP;
748
749     ctx->input_width    = avctx->width;
750     ctx->input_height   = avctx->height;
751     ctx->aligned_width  = FFALIGN(ctx->input_width,  16);
752     ctx->aligned_height = FFALIGN(ctx->input_height, 16);
753     priv->mb_width      = ctx->aligned_width  / 16;
754     priv->mb_height     = ctx->aligned_height / 16;
755
756     if (avctx->bit_rate > 0) {
757         av_log(avctx, AV_LOG_ERROR, "Constant bitrate encoding is not "
758                "supported!\n");
759         return AVERROR_PATCHWELCOME;
760     }
761
762     priv->fixed_qp_p = avctx->global_quality;
763     if (avctx->i_quant_factor > 0.0)
764         priv->fixed_qp_idr = (int)((priv->fixed_qp_p * avctx->i_quant_factor +
765                                     avctx->i_quant_offset) + 0.5);
766     else
767         priv->fixed_qp_idr = priv->fixed_qp_p;
768     if (avctx->b_quant_factor > 0.0)
769         priv->fixed_qp_b = (int)((priv->fixed_qp_p * avctx->b_quant_factor +
770                                   avctx->b_quant_offset) + 0.5);
771     else
772         priv->fixed_qp_b = priv->fixed_qp_p;
773     av_log(avctx, AV_LOG_DEBUG, "QP = %d / %d / %d for IDR / P / B frames.\n",
774            priv->fixed_qp_idr, priv->fixed_qp_p, priv->fixed_qp_b);
775
776     ctx->config_attributes = vaapi_encode_h264_config_attributes;
777     ctx->nb_config_attributes =
778         FF_ARRAY_ELEMS(vaapi_encode_h264_config_attributes);
779
780     ctx->nb_recon_frames = 20;
781
782     return 0;
783 }
784
785 static VAAPIEncodeType vaapi_encode_type_h264 = {
786     .priv_data_size        = sizeof(VAAPIEncodeH264Context),
787
788     .init                  = &vaapi_encode_h264_init_internal,
789
790     .sequence_params_size  = sizeof(VAEncSequenceParameterBufferH264),
791     .init_sequence_params  = &vaapi_encode_h264_init_sequence_params,
792
793     .picture_params_size   = sizeof(VAEncPictureParameterBufferH264),
794     .init_picture_params   = &vaapi_encode_h264_init_picture_params,
795
796     .slice_params_size     = sizeof(VAEncSliceParameterBufferH264),
797     .init_slice_params     = &vaapi_encode_h264_init_slice_params,
798
799     .sequence_header_type  = VAEncPackedHeaderSequence,
800     .write_sequence_header = &vaapi_encode_h264_write_sequence_header,
801
802     .slice_header_type     = VAEncPackedHeaderH264_Slice,
803     .write_slice_header    = &vaapi_encode_h264_write_slice_header,
804 };
805
806 static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
807 {
808     return ff_vaapi_encode_init(avctx, &vaapi_encode_type_h264);
809 }
810
811 static const AVCodecDefault vaapi_encode_h264_defaults[] = {
812     { "profile",        "100" },
813     { "level",          "51"  },
814     { "b",              "0"   },
815     { "bf",             "2"   },
816     { "g",              "120" },
817     { "global_quality", "20"  },
818     { "i_qfactor",      "1.0" },
819     { "i_qoffset",      "0.0" },
820     { "b_qfactor",      "1.2" },
821     { "b_qoffset",      "0.0" },
822     { NULL },
823 };
824
825 static const AVClass vaapi_encode_h264_class = {
826     .class_name = "h264_vaapi",
827     .item_name  = av_default_item_name,
828     .version    = LIBAVUTIL_VERSION_INT,
829 };
830
831 AVCodec ff_h264_vaapi_encoder = {
832     .name           = "h264_vaapi",
833     .long_name      = NULL_IF_CONFIG_SMALL("H.264/AVC (VAAPI)"),
834     .type           = AVMEDIA_TYPE_VIDEO,
835     .id             = AV_CODEC_ID_H264,
836     .priv_data_size = sizeof(VAAPIEncodeContext),
837     .init           = &vaapi_encode_h264_init,
838     .encode2        = &ff_vaapi_encode2,
839     .close          = &ff_vaapi_encode_close,
840     .priv_class     = &vaapi_encode_h264_class,
841     .capabilities   = AV_CODEC_CAP_DELAY,
842     .defaults       = vaapi_encode_h264_defaults,
843     .pix_fmts = (const enum AVPixelFormat[]) {
844         AV_PIX_FMT_VAAPI,
845         AV_PIX_FMT_NONE,
846     },
847 };