libopenh264enc: Return a more sensible error code in some init failure paths
[ffmpeg.git] / libavcodec / vaapi_encode_mjpeg.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include <va/va.h>
20 #include <va/va_enc_jpeg.h>
21
22 #include "libavutil/avassert.h"
23 #include "libavutil/common.h"
24 #include "libavutil/internal.h"
25 #include "libavutil/opt.h"
26 #include "libavutil/pixfmt.h"
27
28 #include "avcodec.h"
29 #include "internal.h"
30 #include "jpegtables.h"
31 #include "mjpeg.h"
32 #include "put_bits.h"
33 #include "vaapi_encode.h"
34
35
36 // Standard JPEG quantisation tables, in zigzag order.
37 static const unsigned char vaapi_encode_mjpeg_quant_luminance[64] = {
38     16,  11,  12,  14,  12,  10,  16,  14,
39     13,  14,  18,  17,  16,  19,  24,  40,
40     26,  24,  22,  22,  24,  49,  35,  37,
41     29,  40,  58,  51,  61,  60,  57,  51,
42     56,  55,  64,  72,  92,  78,  64,  68,
43     87,  69,  55,  56,  80, 109,  81,  87,
44     95,  98, 103, 104, 103,  62,  77, 113,
45    121, 112, 100, 120,  92, 101, 103,  99,
46 };
47 static const unsigned char vaapi_encode_mjpeg_quant_chrominance[64] = {
48     17,  18,  18,  24,  21,  24,  47,  26,
49     26,  47,  99,  66,  56,  66,  99,  99,
50     99,  99,  99,  99,  99,  99,  99,  99,
51     99,  99,  99,  99,  99,  99,  99,  99,
52     99,  99,  99,  99,  99,  99,  99,  99,
53     99,  99,  99,  99,  99,  99,  99,  99,
54     99,  99,  99,  99,  99,  99,  99,  99,
55     99,  99,  99,  99,  99,  99,  99,  99,
56 };
57
58 typedef struct VAAPIEncodeMJPEGContext {
59     int quality;
60     int component_subsample_h[3];
61     int component_subsample_v[3];
62
63     VAQMatrixBufferJPEG quant_tables;
64     VAHuffmanTableBufferJPEGBaseline huffman_tables;
65 } VAAPIEncodeMJPEGContext;
66
67 static av_cold void vaapi_encode_mjpeg_copy_huffman(unsigned char *dst_lengths,
68                                                     unsigned char *dst_values,
69                                                     const unsigned char *src_lengths,
70                                                     const unsigned char *src_values)
71 {
72     int i, mt;
73
74     ++src_lengths;
75
76     mt = 0;
77     for (i = 0; i < 16; i++)
78         mt += (dst_lengths[i] = src_lengths[i]);
79
80     for (i = 0; i < mt; i++)
81         dst_values[i] = src_values[i];
82 }
83
84 static av_cold void vaapi_encode_mjpeg_init_tables(AVCodecContext *avctx)
85 {
86     VAAPIEncodeContext                *ctx = avctx->priv_data;
87     VAAPIEncodeMJPEGContext          *priv = ctx->priv_data;
88     VAQMatrixBufferJPEG             *quant = &priv->quant_tables;
89     VAHuffmanTableBufferJPEGBaseline *huff = &priv->huffman_tables;
90     int i;
91
92     quant->load_lum_quantiser_matrix = 1;
93     quant->load_chroma_quantiser_matrix = 1;
94
95     for (i = 0; i < 64; i++) {
96         quant->lum_quantiser_matrix[i] =
97             vaapi_encode_mjpeg_quant_luminance[i];
98         quant->chroma_quantiser_matrix[i] =
99             vaapi_encode_mjpeg_quant_chrominance[i];
100     }
101
102     huff->load_huffman_table[0] = 1;
103     vaapi_encode_mjpeg_copy_huffman(huff->huffman_table[0].num_dc_codes,
104                                     huff->huffman_table[0].dc_values,
105                                     avpriv_mjpeg_bits_dc_luminance,
106                                     avpriv_mjpeg_val_dc);
107     vaapi_encode_mjpeg_copy_huffman(huff->huffman_table[0].num_ac_codes,
108                                     huff->huffman_table[0].ac_values,
109                                     avpriv_mjpeg_bits_ac_luminance,
110                                     avpriv_mjpeg_val_ac_luminance);
111     memset(huff->huffman_table[0].pad, 0, sizeof(huff->huffman_table[0].pad));
112
113     huff->load_huffman_table[1] = 1;
114     vaapi_encode_mjpeg_copy_huffman(huff->huffman_table[1].num_dc_codes,
115                                     huff->huffman_table[1].dc_values,
116                                     avpriv_mjpeg_bits_dc_chrominance,
117                                     avpriv_mjpeg_val_dc);
118     vaapi_encode_mjpeg_copy_huffman(huff->huffman_table[1].num_ac_codes,
119                                     huff->huffman_table[1].ac_values,
120                                     avpriv_mjpeg_bits_ac_chrominance,
121                                     avpriv_mjpeg_val_ac_chrominance);
122     memset(huff->huffman_table[1].pad, 0, sizeof(huff->huffman_table[1].pad));
123 }
124
125 static void vaapi_encode_mjpeg_write_marker(PutBitContext *pbc, int marker)
126 {
127     put_bits(pbc, 8, 0xff);
128     put_bits(pbc, 8, marker);
129 }
130
131 static int vaapi_encode_mjpeg_write_image_header(AVCodecContext *avctx,
132                                                  VAAPIEncodePicture *pic,
133                                                  VAAPIEncodeSlice *slice,
134                                                  char *data, size_t *data_len)
135 {
136     VAAPIEncodeContext               *ctx = avctx->priv_data;
137     VAEncPictureParameterBufferJPEG *vpic = pic->codec_picture_params;
138     VAEncSliceParameterBufferJPEG *vslice = slice->codec_slice_params;
139     VAAPIEncodeMJPEGContext         *priv = ctx->priv_data;
140     PutBitContext pbc;
141     int t, i, quant_scale;
142
143     init_put_bits(&pbc, data, *data_len);
144
145     vaapi_encode_mjpeg_write_marker(&pbc, SOI);
146
147     // Quantisation table coefficients are scaled for quality by the driver,
148     // so we also need to do it ourselves here so that headers match.
149     if (priv->quality < 50)
150         quant_scale = 5000 / priv->quality;
151     else
152         quant_scale = 200 - 2 * priv->quality;
153
154     for (t = 0; t < 2; t++) {
155         int q;
156
157         vaapi_encode_mjpeg_write_marker(&pbc, DQT);
158
159         put_bits(&pbc, 16, 3 + 64); // Lq
160         put_bits(&pbc, 4, 0); // Pq
161         put_bits(&pbc, 4, t); // Tq
162
163         for (i = 0; i < 64; i++) {
164             q = i[t ? priv->quant_tables.chroma_quantiser_matrix
165                     : priv->quant_tables.lum_quantiser_matrix];
166             q = (q * quant_scale) / 100;
167             if (q < 1)   q = 1;
168             if (q > 255) q = 255;
169             put_bits(&pbc, 8, q);
170         }
171     }
172
173     vaapi_encode_mjpeg_write_marker(&pbc, SOF0);
174
175     put_bits(&pbc, 16, 8 + 3 * vpic->num_components); // Lf
176     put_bits(&pbc, 8,  vpic->sample_bit_depth); // P
177     put_bits(&pbc, 16, vpic->picture_height);   // Y
178     put_bits(&pbc, 16, vpic->picture_width);    // X
179     put_bits(&pbc, 8,  vpic->num_components);   // Nf
180
181     for (i = 0; i < vpic->num_components; i++) {
182         put_bits(&pbc, 8, vpic->component_id[i]); // Ci
183         put_bits(&pbc, 4, priv->component_subsample_h[i]); // Hi
184         put_bits(&pbc, 4, priv->component_subsample_v[i]); // Vi
185         put_bits(&pbc, 8, vpic->quantiser_table_selector[i]); // Tqi
186     }
187
188     for (t = 0; t < 4; t++) {
189         int mt;
190         unsigned char *lengths, *values;
191
192         vaapi_encode_mjpeg_write_marker(&pbc, DHT);
193
194         if ((t & 1) == 0) {
195             lengths = priv->huffman_tables.huffman_table[t / 2].num_dc_codes;
196             values  = priv->huffman_tables.huffman_table[t / 2].dc_values;
197         } else {
198             lengths = priv->huffman_tables.huffman_table[t / 2].num_ac_codes;
199             values  = priv->huffman_tables.huffman_table[t / 2].ac_values;
200         }
201
202         mt = 0;
203         for (i = 0; i < 16; i++)
204             mt += lengths[i];
205
206         put_bits(&pbc, 16, 2 + 17 + mt); // Lh
207         put_bits(&pbc, 4, t & 1); // Tc
208         put_bits(&pbc, 4, t / 2); // Th
209
210         for (i = 0; i < 16; i++)
211             put_bits(&pbc, 8, lengths[i]);
212         for (i = 0; i < mt; i++)
213             put_bits(&pbc, 8, values[i]);
214     }
215
216     vaapi_encode_mjpeg_write_marker(&pbc, SOS);
217
218     av_assert0(vpic->num_components == vslice->num_components);
219
220     put_bits(&pbc, 16, 6 + 2 * vslice->num_components); // Ls
221     put_bits(&pbc, 8,  vslice->num_components); // Ns
222
223     for (i = 0; i < vslice->num_components; i++) {
224         put_bits(&pbc, 8, vslice->components[i].component_selector); // Csj
225         put_bits(&pbc, 4, vslice->components[i].dc_table_selector);  // Tdj
226         put_bits(&pbc, 4, vslice->components[i].ac_table_selector);  // Taj
227     }
228
229     put_bits(&pbc, 8, 0); // Ss
230     put_bits(&pbc, 8, 63); // Se
231     put_bits(&pbc, 4, 0); // Ah
232     put_bits(&pbc, 4, 0); // Al
233
234     *data_len = put_bits_count(&pbc);
235     flush_put_bits(&pbc);
236
237     return 0;
238 }
239
240 static int vaapi_encode_mjpeg_write_extra_buffer(AVCodecContext *avctx,
241                                                  VAAPIEncodePicture *pic,
242                                                  int index, int *type,
243                                                  char *data, size_t *data_len)
244 {
245     VAAPIEncodeContext       *ctx = avctx->priv_data;
246     VAAPIEncodeMJPEGContext *priv = ctx->priv_data;
247
248     if (index == 0) {
249         // Write quantisation tables.
250         if (*data_len < sizeof(priv->quant_tables))
251             return AVERROR(EINVAL);
252         *type = VAQMatrixBufferType;
253         memcpy(data, &priv->quant_tables,
254                *data_len = sizeof(priv->quant_tables));
255
256     } else if (index == 1) {
257         // Write huffman tables.
258         if (*data_len < sizeof(priv->huffman_tables))
259             return AVERROR(EINVAL);
260         *type = VAHuffmanTableBufferType;
261         memcpy(data, &priv->huffman_tables,
262                *data_len = sizeof(priv->huffman_tables));
263
264     } else {
265         return AVERROR_EOF;
266     }
267     return 0;
268 }
269
270 static int vaapi_encode_mjpeg_init_picture_params(AVCodecContext *avctx,
271                                                   VAAPIEncodePicture *pic)
272 {
273     VAAPIEncodeContext               *ctx = avctx->priv_data;
274     VAEncPictureParameterBufferJPEG *vpic = pic->codec_picture_params;
275     VAAPIEncodeMJPEGContext         *priv = ctx->priv_data;
276
277     vpic->reconstructed_picture = pic->recon_surface;
278     vpic->coded_buf = pic->output_buffer;
279
280     vpic->picture_width  = ctx->input_width;
281     vpic->picture_height = ctx->input_height;
282
283     vpic->pic_flags.bits.profile      = 0;
284     vpic->pic_flags.bits.progressive  = 0;
285     vpic->pic_flags.bits.huffman      = 1;
286     vpic->pic_flags.bits.interleaved  = 0;
287     vpic->pic_flags.bits.differential = 0;
288
289     vpic->sample_bit_depth = 8;
290     vpic->num_scan = 1;
291
292     vpic->num_components = 3;
293
294     vpic->component_id[0] = 1;
295     vpic->component_id[1] = 2;
296     vpic->component_id[2] = 3;
297
298     priv->component_subsample_h[0] = 2;
299     priv->component_subsample_v[0] = 2;
300     priv->component_subsample_h[1] = 1;
301     priv->component_subsample_v[1] = 1;
302     priv->component_subsample_h[2] = 1;
303     priv->component_subsample_v[2] = 1;
304
305     vpic->quantiser_table_selector[0] = 0;
306     vpic->quantiser_table_selector[1] = 1;
307     vpic->quantiser_table_selector[2] = 1;
308
309     vpic->quality = priv->quality;
310
311     pic->nb_slices = 1;
312
313     return 0;
314 }
315
316 static int vaapi_encode_mjpeg_init_slice_params(AVCodecContext *avctx,
317                                                 VAAPIEncodePicture *pic,
318                                                 VAAPIEncodeSlice *slice)
319 {
320     VAEncPictureParameterBufferJPEG *vpic = pic->codec_picture_params;
321     VAEncSliceParameterBufferJPEG *vslice = slice->codec_slice_params;
322     int i;
323
324     vslice->restart_interval = 0;
325
326     vslice->num_components = vpic->num_components;
327     for (i = 0; i < vslice->num_components; i++) {
328         vslice->components[i].component_selector = i + 1;
329         vslice->components[i].dc_table_selector = (i > 0);
330         vslice->components[i].ac_table_selector = (i > 0);
331     }
332
333     return 0;
334 }
335
336 static av_cold int vaapi_encode_mjpeg_init_internal(AVCodecContext *avctx)
337 {
338     static const VAConfigAttrib default_config_attributes[] = {
339         { .type  = VAConfigAttribRTFormat,
340           .value = VA_RT_FORMAT_YUV420 },
341         { .type  = VAConfigAttribEncPackedHeaders,
342           .value = VA_ENC_PACKED_HEADER_SEQUENCE },
343     };
344
345     VAAPIEncodeContext       *ctx = avctx->priv_data;
346     VAAPIEncodeMJPEGContext *priv = ctx->priv_data;
347     int i;
348
349     ctx->va_profile    = VAProfileJPEGBaseline;
350     ctx->va_entrypoint = VAEntrypointEncPicture;
351
352     ctx->input_width    = avctx->width;
353     ctx->input_height   = avctx->height;
354     ctx->aligned_width  = FFALIGN(ctx->input_width,  8);
355     ctx->aligned_height = FFALIGN(ctx->input_height, 8);
356
357     for (i = 0; i < FF_ARRAY_ELEMS(default_config_attributes); i++) {
358         ctx->config_attributes[ctx->nb_config_attributes++] =
359             default_config_attributes[i];
360     }
361
362     priv->quality = avctx->global_quality;
363     if (priv->quality < 1 || priv->quality > 100) {
364         av_log(avctx, AV_LOG_ERROR, "Invalid quality value %d "
365                "(must be 1-100).\n", priv->quality);
366         return AVERROR(EINVAL);
367     }
368
369     vaapi_encode_mjpeg_init_tables(avctx);
370
371     return 0;
372 }
373
374 static VAAPIEncodeType vaapi_encode_type_mjpeg = {
375     .priv_data_size        = sizeof(VAAPIEncodeMJPEGContext),
376
377     .init                  = &vaapi_encode_mjpeg_init_internal,
378
379     .picture_params_size   = sizeof(VAEncPictureParameterBufferJPEG),
380     .init_picture_params   = &vaapi_encode_mjpeg_init_picture_params,
381
382     .slice_params_size     = sizeof(VAEncSliceParameterBufferJPEG),
383     .init_slice_params     = &vaapi_encode_mjpeg_init_slice_params,
384
385     .slice_header_type     = VAEncPackedHeaderRawData,
386     .write_slice_header    = &vaapi_encode_mjpeg_write_image_header,
387
388     .write_extra_buffer    = &vaapi_encode_mjpeg_write_extra_buffer,
389 };
390
391 static av_cold int vaapi_encode_mjpeg_init(AVCodecContext *avctx)
392 {
393     return ff_vaapi_encode_init(avctx, &vaapi_encode_type_mjpeg);
394 }
395
396 static const AVCodecDefault vaapi_encode_mjpeg_defaults[] = {
397     { "global_quality", "80" },
398     { NULL },
399 };
400
401 static const AVClass vaapi_encode_mjpeg_class = {
402     .class_name = "mjpeg_vaapi",
403     .item_name  = av_default_item_name,
404     .version    = LIBAVUTIL_VERSION_INT,
405 };
406
407 AVCodec ff_mjpeg_vaapi_encoder = {
408     .name           = "mjpeg_vaapi",
409     .long_name      = NULL_IF_CONFIG_SMALL("MJPEG (VAAPI)"),
410     .type           = AVMEDIA_TYPE_VIDEO,
411     .id             = AV_CODEC_ID_MJPEG,
412     .priv_data_size = sizeof(VAAPIEncodeContext),
413     .init           = &vaapi_encode_mjpeg_init,
414     .encode2        = &ff_vaapi_encode2,
415     .close          = &ff_vaapi_encode_close,
416     .priv_class     = &vaapi_encode_mjpeg_class,
417     .defaults       = vaapi_encode_mjpeg_defaults,
418     .pix_fmts = (const enum AVPixelFormat[]) {
419         AV_PIX_FMT_VAAPI,
420         AV_PIX_FMT_NONE,
421     },
422 };