avcodec/ljpegenc: fix mem allocation failure return code encode_picture_lossless()
[ffmpeg.git] / libavcodec / ljpegenc.c
index 56336cb..7e536a8 100644 (file)
  */
 
 #include "avcodec.h"
-#include "dsputil.h"
+#include "internal.h"
 #include "mpegvideo.h"
 #include "mjpeg.h"
 #include "mjpegenc.h"
 
 
-static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, int buf_size, void *data){
+static int encode_picture_lossless(AVCodecContext *avctx, AVPacket *pkt,
+                                   const AVFrame *pict, int *got_packet)
+{
     MpegEncContext * const s = avctx->priv_data;
     MJpegContext * const m = s->mjpeg_ctx;
-    AVFrame *pict = data;
     const int width= s->width;
     const int height= s->height;
-    AVFrame * const p= (AVFrame*)&s->current_picture;
+    AVFrame * const p = &s->current_picture.f;
     const int predictor= avctx->prediction_method+1;
+    const int mb_width  = (width  + s->mjpeg_hsample[0] - 1) / s->mjpeg_hsample[0];
+    const int mb_height = (height + s->mjpeg_vsample[0] - 1) / s->mjpeg_vsample[0];
+    int ret, max_pkt_size = FF_MIN_BUFFER_SIZE;
+
+    if (avctx->pix_fmt == AV_PIX_FMT_BGRA)
+        max_pkt_size += width * height * 3 * 4;
+    else {
+        max_pkt_size += mb_width * mb_height * 3 * 4
+                        * s->mjpeg_hsample[0] * s->mjpeg_vsample[0];
+    }
+
+    if (!s->rd_scratchpad) {
+        int alloc_size = FFALIGN(FFABS(pict->linesize[0]) + 64, 32);
+        s->me.scratchpad =
+        s->rd_scratchpad = av_mallocz(alloc_size * 4 * 16 * 2);
+        if (!s->rd_scratchpad) {
+            av_log(avctx, AV_LOG_ERROR, "failed to allocate context scratch buffers.\n");
+            return AVERROR(ENOMEM);
+        }
+    }
+
+    if ((ret = ff_alloc_packet2(avctx, pkt, max_pkt_size)) < 0)
+        return ret;
 
-    init_put_bits(&s->pb, buf, buf_size);
+    init_put_bits(&s->pb, pkt->data, pkt->size);
 
-    *p = *pict;
-    p->pict_type= FF_I_TYPE;
+    av_frame_unref(p);
+    ret = av_frame_ref(p, pict);
+    if (ret < 0)
+        return ret;
+    p->pict_type= AV_PICTURE_TYPE_I;
     p->key_frame= 1;
 
-    ff_mjpeg_encode_picture_header(s);
+    ff_mjpeg_encode_picture_header(avctx, &s->pb, &s->intra_scantable,
+                                   s->intra_matrix);
 
     s->header_bits= put_bits_count(&s->pb);
 
-    if(avctx->pix_fmt == PIX_FMT_BGRA){
+    if(avctx->pix_fmt == AV_PIX_FMT_BGR0
+        || avctx->pix_fmt == AV_PIX_FMT_BGRA
+        || avctx->pix_fmt == AV_PIX_FMT_BGR24){
         int x, y, i;
         const int linesize= p->linesize[0];
         uint16_t (*buffer)[4]= (void *) s->rd_scratchpad;
@@ -79,9 +109,15 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
                 top[i]= left[i]= topleft[i]= buffer[0][i];
             }
             for(x = 0; x < width; x++) {
+                if(avctx->pix_fmt == AV_PIX_FMT_BGR24){
+                    buffer[x][1] = ptr[3*x+0] - ptr[3*x+1] + 0x100;
+                    buffer[x][2] = ptr[3*x+2] - ptr[3*x+1] + 0x100;
+                    buffer[x][0] = (ptr[3*x+0] + 2*ptr[3*x+1] + ptr[3*x+2])>>2;
+                }else{
                 buffer[x][1] = ptr[4*x+0] - ptr[4*x+1] + 0x100;
                 buffer[x][2] = ptr[4*x+2] - ptr[4*x+1] + 0x100;
                 buffer[x][0] = (ptr[4*x+0] + 2*ptr[4*x+1] + ptr[4*x+2])>>2;
+                }
 
                 for(i=0;i<3;i++) {
                     int pred, diff;
@@ -96,16 +132,14 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
                     diff= ((left[i] - pred + 0x100)&0x1FF) - 0x100;
 
                     if(i==0)
-                        ff_mjpeg_encode_dc(s, diff, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
+                        ff_mjpeg_encode_dc(&s->pb, diff, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
                     else
-                        ff_mjpeg_encode_dc(s, diff, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
+                        ff_mjpeg_encode_dc(&s->pb, diff, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
                 }
             }
         }
     }else{
         int mb_x, mb_y, i;
-        const int mb_width  = (width  + s->mjpeg_hsample[0] - 1) / s->mjpeg_hsample[0];
-        const int mb_height = (height + s->mjpeg_vsample[0] - 1) / s->mjpeg_vsample[0];
 
         for(mb_y = 0; mb_y < mb_height; mb_y++) {
             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < mb_width * 4 * 3 * s->mjpeg_hsample[0] * s->mjpeg_vsample[0]){
@@ -141,9 +175,9 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
                                 }
 
                                 if(i==0)
-                                    ff_mjpeg_encode_dc(s, *ptr - pred, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
+                                    ff_mjpeg_encode_dc(&s->pb, *ptr - pred, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
                                 else
-                                    ff_mjpeg_encode_dc(s, *ptr - pred, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
+                                    ff_mjpeg_encode_dc(&s->pb, *ptr - pred, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
                             }
                         }
                     }
@@ -153,20 +187,19 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
                         int x, y, h, v, linesize;
                         h = s->mjpeg_hsample[i];
                         v = s->mjpeg_vsample[i];
-                        linesize= p->linesize[i];
+                        linesize = pict->linesize[i];
 
                         for(y=0; y<v; y++){
                             for(x=0; x<h; x++){
                                 int pred;
 
-                                ptr = p->data[i] + (linesize * (v * mb_y + y)) + (h * mb_x + x); //FIXME optimize this crap
-//printf("%d %d %d %d %8X\n", mb_x, mb_y, x, y, ptr);
+                                ptr = pict->data[i] + (linesize * (v * mb_y + y)) + (h * mb_x + x); //FIXME optimize this crap
                                 PREDICT(pred, ptr[-linesize-1], ptr[-linesize], ptr[-1], predictor);
 
                                 if(i==0)
-                                    ff_mjpeg_encode_dc(s, *ptr - pred, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
+                                    ff_mjpeg_encode_dc(&s->pb, *ptr - pred, m->huff_size_dc_luminance, m->huff_code_dc_luminance); //FIXME ugly
                                 else
-                                    ff_mjpeg_encode_dc(s, *ptr - pred, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
+                                    ff_mjpeg_encode_dc(&s->pb, *ptr - pred, m->huff_size_dc_chrominance, m->huff_code_dc_chrominance);
                             }
                         }
                     }
@@ -176,23 +209,34 @@ static int encode_picture_lossless(AVCodecContext *avctx, unsigned char *buf, in
     }
 
     emms_c();
+    av_assert0(s->esc_pos == s->header_bits >> 3);
 
-    ff_mjpeg_encode_picture_trailer(s);
+    ff_mjpeg_encode_stuffing(s);
+    ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
     s->picture_number++;
 
     flush_put_bits(&s->pb);
-    return put_bits_ptr(&s->pb) - s->pb.buf;
+    pkt->size   = put_bits_ptr(&s->pb) - s->pb.buf;
+    pkt->flags |= AV_PKT_FLAG_KEY;
+    *got_packet = 1;
+
+    return 0;
 //    return (put_bits_count(&f->pb)+7)/8;
 }
 
 
 AVCodec ff_ljpeg_encoder = { //FIXME avoid MPV_* lossless JPEG should not need them
-    "ljpeg",
-    AVMEDIA_TYPE_VIDEO,
-    CODEC_ID_LJPEG,
-    sizeof(MpegEncContext),
-    MPV_encode_init,
-    encode_picture_lossless,
-    MPV_encode_end,
-    .long_name = NULL_IF_CONFIG_SMALL("Lossless JPEG"),
+    .name           = "ljpeg",
+    .long_name      = NULL_IF_CONFIG_SMALL("Lossless JPEG"),
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_LJPEG,
+    .priv_data_size = sizeof(MpegEncContext),
+    .init           = ff_MPV_encode_init,
+    .encode2        = encode_picture_lossless,
+    .close          = ff_MPV_encode_end,
+    .pix_fmts       = (const enum AVPixelFormat[]){
+        AV_PIX_FMT_BGR24, AV_PIX_FMT_BGRA, AV_PIX_FMT_BGR0,
+        AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ444P, AV_PIX_FMT_YUVJ422P,
+        AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV422P,
+        AV_PIX_FMT_NONE},
 };