lavc/j2kenc: Support AV_PIX_FMT_PAL8.
[ffmpeg.git] / libavcodec / fmvc.c
1 /*
2  * FM Screen Capture Codec decoder
3  *
4  * Copyright (c) 2017 Paul B Mahol
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26
27 #include "avcodec.h"
28 #include "bytestream.h"
29 #include "internal.h"
30
31 #define BLOCK_HEIGHT 112u
32 #define BLOCK_WIDTH  84u
33
34 typedef struct InterBlock {
35     int      w, h;
36     int      size;
37     int      xor;
38 } InterBlock;
39
40 typedef struct FMVCContext {
41     GetByteContext  gb;
42     PutByteContext  pb;
43     uint8_t        *buffer;
44     size_t          buffer_size;
45     uint8_t        *pbuffer;
46     size_t          pbuffer_size;
47     ptrdiff_t       stride;
48     int             bpp;
49     int             yb, xb;
50     InterBlock     *blocks;
51     unsigned        nb_blocks;
52 } FMVCContext;
53
54 static int decode_type2(GetByteContext *gb, PutByteContext *pb)
55 {
56     unsigned repeat = 0, first = 1, opcode = 0;
57     int i, len, pos;
58
59     while (bytestream2_get_bytes_left(gb) > 0) {
60         GetByteContext gbc;
61
62         while (bytestream2_get_bytes_left(gb) > 0) {
63             if (first) {
64                 first = 0;
65                 if (bytestream2_peek_byte(gb) > 17) {
66                     len = bytestream2_get_byte(gb) - 17;
67                     if (len < 4) {
68                         do {
69                             bytestream2_put_byte(pb, bytestream2_get_byte(gb));
70                             --len;
71                         } while (len);
72                         opcode = bytestream2_peek_byte(gb);
73                         continue;
74                     } else {
75                         do {
76                             bytestream2_put_byte(pb, bytestream2_get_byte(gb));
77                             --len;
78                         } while (len);
79                         opcode = bytestream2_peek_byte(gb);
80                         if (opcode < 0x10) {
81                             bytestream2_skip(gb, 1);
82                             pos = - (opcode >> 2) - 4 * bytestream2_get_byte(gb) - 2049;
83
84                             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
85                             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
86
87                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
88                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
89                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
90                             len = opcode & 3;
91                             if (!len) {
92                                 repeat = 1;
93                             } else {
94                                 do {
95                                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
96                                     --len;
97                                 } while (len);
98                                 opcode = bytestream2_peek_byte(gb);
99                             }
100                             continue;
101                         }
102                     }
103                     repeat = 0;
104                 }
105                 repeat = 1;
106             }
107             if (repeat) {
108                 repeat = 0;
109                 opcode = bytestream2_peek_byte(gb);
110                 if (opcode < 0x10) {
111                     bytestream2_skip(gb, 1);
112                     if (!opcode) {
113                         if (!bytestream2_peek_byte(gb)) {
114                             do {
115                                 bytestream2_skip(gb, 1);
116                                 opcode += 255;
117                             } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
118                         }
119                         opcode += bytestream2_get_byte(gb) + 15;
120                     }
121                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
122                     for (i = opcode - 1; i > 0; --i)
123                         bytestream2_put_byte(pb, bytestream2_get_byte(gb));
124                     opcode = bytestream2_peek_byte(gb);
125                     if (opcode < 0x10) {
126                         bytestream2_skip(gb, 1);
127                         pos = - (opcode >> 2) - 4 * bytestream2_get_byte(gb) - 2049;
128
129                         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
130                         bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
131
132                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
133                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
134                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
135                         len = opcode & 3;
136                         if (!len) {
137                             repeat = 1;
138                         } else {
139                             do {
140                                 bytestream2_put_byte(pb, bytestream2_get_byte(gb));
141                                 --len;
142                             } while (len);
143                             opcode = bytestream2_peek_byte(gb);
144                         }
145                         continue;
146                     }
147                 }
148             }
149
150             if (opcode >= 0x40) {
151                 bytestream2_skip(gb, 1);
152                 pos = - ((opcode >> 2) & 7) - 1 - 8 * bytestream2_get_byte(gb);
153                 len =    (opcode >> 5)      - 1;
154
155                 bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
156                 bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
157
158                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
159                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
160                 do {
161                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
162                     --len;
163                 } while (len);
164
165                 len = opcode & 3;
166
167                 if (!len) {
168                     repeat = 1;
169                 } else {
170                     do {
171                         bytestream2_put_byte(pb, bytestream2_get_byte(gb));
172                         --len;
173                     } while (len);
174                     opcode = bytestream2_peek_byte(gb);
175                 }
176                 continue;
177             } else if (opcode < 0x20) {
178                 break;
179             }
180             len = opcode & 0x1F;
181             bytestream2_skip(gb, 1);
182             if (!len) {
183                 if (!bytestream2_peek_byte(gb)) {
184                     do {
185                         bytestream2_skip(gb, 1);
186                         len += 255;
187                     } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
188                 }
189                 len += bytestream2_get_byte(gb) + 31;
190             }
191             i = bytestream2_get_le16(gb);
192             pos = - (i >> 2) - 1;
193
194             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
195             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
196
197             if (len < 6 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
198                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
199                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
200                 do {
201                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
202                     --len;
203                 } while (len);
204             } else {
205                 bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
206                 for (len = len - 2; len; --len)
207                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
208             }
209             len = i & 3;
210             if (!len) {
211                 repeat = 1;
212             } else {
213                 do {
214                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
215                     --len;
216                 } while (len);
217                 opcode = bytestream2_peek_byte(gb);
218             }
219         }
220         bytestream2_skip(gb, 1);
221         if (opcode < 0x10) {
222             pos = -(opcode >> 2) - 1 - 4 * bytestream2_get_byte(gb);
223
224             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
225             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
226
227             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
228             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
229             len = opcode & 3;
230             if (!len) {
231                 repeat = 1;
232             } else {
233                 do {
234                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
235                     --len;
236                 } while (len);
237                 opcode = bytestream2_peek_byte(gb);
238             }
239             continue;
240         }
241         len = opcode & 7;
242         if (!len) {
243             if (!bytestream2_peek_byte(gb)) {
244                 do {
245                     bytestream2_skip(gb, 1);
246                     len += 255;
247                 } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
248             }
249             len += bytestream2_get_byte(gb) + 7;
250         }
251         i = bytestream2_get_le16(gb);
252         pos = bytestream2_tell_p(pb) - 2048 * (opcode & 8);
253         pos = pos - (i >> 2);
254         if (pos == bytestream2_tell_p(pb))
255             break;
256
257         pos = pos - 0x4000;
258         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
259         bytestream2_seek(&gbc, pos, SEEK_SET);
260
261         if (len < 6 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
262             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
263             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
264             do {
265                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
266                 --len;
267             } while (len);
268         } else {
269             bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
270             for (len = len - 2; len; --len)
271                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
272         }
273
274         len = i & 3;
275         if (!len) {
276             repeat = 1;
277         } else {
278             do {
279                 bytestream2_put_byte(pb, bytestream2_get_byte(gb));
280                 --len;
281             } while (len);
282             opcode = bytestream2_peek_byte(gb);
283         }
284     }
285
286     return 0;
287 }
288
289 static int decode_type1(GetByteContext *gb, PutByteContext *pb)
290 {
291     unsigned opcode = 0, len;
292     int high = 0;
293     int i, pos;
294
295     while (bytestream2_get_bytes_left(gb) > 0) {
296         GetByteContext gbc;
297
298         while (bytestream2_get_bytes_left(gb) > 0) {
299             while (bytestream2_get_bytes_left(gb) > 0) {
300                 opcode = bytestream2_get_byte(gb);
301                 high = opcode >= 0x20;
302                 if (high)
303                     break;
304                 if (opcode)
305                     break;
306                 opcode = bytestream2_get_byte(gb);
307                 if (opcode < 0xF8) {
308                     opcode += 32;
309                     break;
310                 }
311                 i = opcode - 0xF8;
312                 if (i) {
313                     len = 256;
314                     do {
315                         len *= 2;
316                         --i;
317                     } while (i);
318                 } else {
319                     len = 280;
320                 }
321                 do {
322                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
323                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
324                     len -= 8;
325                 } while (len && bytestream2_get_bytes_left(gb) > 0);
326             }
327
328             if (!high) {
329                 do {
330                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
331                     --opcode;
332                 } while (opcode && bytestream2_get_bytes_left(gb) > 0);
333
334                 while (bytestream2_get_bytes_left(gb) > 0) {
335                     GetByteContext gbc;
336
337                     opcode = bytestream2_get_byte(gb);
338                     if (opcode >= 0x20)
339                         break;
340                     bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
341
342                     pos = -(opcode | 32 * bytestream2_get_byte(gb)) - 1;
343                     bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
344                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
345                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
346                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
347                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
348                 }
349             }
350             high = 0;
351             if (opcode < 0x40)
352                 break;
353             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
354             pos = (-((opcode & 0x1F) | 32 * bytestream2_get_byte(gb)) - 1);
355             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
356             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
357             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
358             len = (opcode >> 5) - 1;
359             do {
360                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
361                 --len;
362             } while (len && bytestream2_get_bytes_left(&gbc) > 0);
363         }
364         len = opcode & 0x1F;
365         if (!len) {
366             if (!bytestream2_peek_byte(gb)) {
367                 do {
368                     bytestream2_skip(gb, 1);
369                     len += 255;
370                 } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
371             }
372             len += bytestream2_get_byte(gb) + 31;
373         }
374         pos = -bytestream2_get_byte(gb);
375         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
376         bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos - (bytestream2_get_byte(gb) << 8), SEEK_SET);
377         if (bytestream2_tell_p(pb) == bytestream2_tell(&gbc))
378             break;
379         if (len < 5 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
380             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
381             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
382             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
383         } else {
384             bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
385             len--;
386         }
387         do {
388             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
389             len--;
390         } while (len && bytestream2_get_bytes_left(&gbc) > 0);
391     }
392
393     return 0;
394 }
395
396 static int decode_frame(AVCodecContext *avctx, void *data,
397                         int *got_frame, AVPacket *avpkt)
398 {
399     FMVCContext *s = avctx->priv_data;
400     GetByteContext *gb = &s->gb;
401     PutByteContext *pb = &s->pb;
402     AVFrame *frame = data;
403     int ret, y, x;
404
405     if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
406         return ret;
407
408     bytestream2_init(gb, avpkt->data, avpkt->size);
409     bytestream2_skip(gb, 2);
410
411     frame->key_frame = !!bytestream2_get_le16(gb);
412     frame->pict_type = frame->key_frame ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_P;
413
414     if (frame->key_frame) {
415         const uint8_t *src;
416         unsigned type, size;
417         uint8_t *dst;
418
419         type = bytestream2_get_le16(gb);
420         size = bytestream2_get_le16(gb);
421         if (size > bytestream2_get_bytes_left(gb))
422             return AVERROR_INVALIDDATA;
423
424         bytestream2_init_writer(pb, s->buffer, s->buffer_size);
425         if (type == 1) {
426             decode_type1(gb, pb);
427         } else if (type == 2){
428             decode_type2(gb, pb);
429         } else {
430             avpriv_report_missing_feature(avctx, "Compression type %d", type);
431             return AVERROR_PATCHWELCOME;
432         }
433
434         src = s->buffer;
435         dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
436         for (y = 0; y < avctx->height; y++) {
437             memcpy(dst, src, avctx->width * s->bpp);
438             dst -= frame->linesize[0];
439             src += s->stride * 4;
440         }
441     } else {
442         unsigned block, nb_blocks;
443         int type, k, l;
444         uint8_t *ssrc, *ddst;
445         const uint32_t *src;
446         uint32_t *dst;
447
448         for (block = 0; block < s->nb_blocks; block++)
449             s->blocks[block].xor = 0;
450
451         nb_blocks = bytestream2_get_le16(gb);
452         if (nb_blocks > s->nb_blocks)
453             return AVERROR_INVALIDDATA;
454
455         bytestream2_init_writer(pb, s->pbuffer, s->pbuffer_size);
456
457         type = bytestream2_get_le16(gb);
458         for (block = 0; block < nb_blocks; block++) {
459             unsigned size, offset;
460             int start = 0;
461
462             offset = bytestream2_get_le16(gb);
463             if (offset >= s->nb_blocks)
464                 return AVERROR_INVALIDDATA;
465
466             size = bytestream2_get_le16(gb);
467             if (size > bytestream2_get_bytes_left(gb))
468                 return AVERROR_INVALIDDATA;
469
470             start = bytestream2_tell_p(pb);
471             if (type == 1) {
472                 decode_type1(gb, pb);
473             } else if (type == 2){
474                 decode_type2(gb, pb);
475             } else {
476                 avpriv_report_missing_feature(avctx, "Compression type %d", type);
477                 return AVERROR_PATCHWELCOME;
478             }
479
480             if (s->blocks[offset].size * 4 != bytestream2_tell_p(pb) - start)
481                 return AVERROR_INVALIDDATA;
482
483             s->blocks[offset].xor = 1;
484         }
485
486         src = (const uint32_t *)s->pbuffer;
487         dst = (uint32_t *)s->buffer;
488
489         for (block = 0, y = 0; y < s->yb; y++) {
490             int block_h = s->blocks[block].h;
491             uint32_t *rect = dst;
492
493             for (x = 0; x < s->xb; x++) {
494                 int block_w = s->blocks[block].w;
495                 uint32_t *row = dst;
496
497                 block_h = s->blocks[block].h;
498                 if (s->blocks[block].xor) {
499                     for (k = 0; k < block_h; k++) {
500                         uint32_t *column = dst;
501                         for (l = 0; l < block_w; l++)
502                             *dst++ ^= *src++;
503                         dst = &column[s->stride];
504                     }
505                 }
506                 dst = &row[block_w];
507                 ++block;
508             }
509             dst = &rect[block_h * s->stride];
510         }
511
512         ssrc = s->buffer;
513         ddst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
514         for (y = 0; y < avctx->height; y++) {
515             memcpy(ddst, ssrc, avctx->width * s->bpp);
516             ddst -= frame->linesize[0];
517             ssrc += s->stride * 4;
518         }
519     }
520
521     *got_frame = 1;
522
523     return avpkt->size;
524 }
525
526 static av_cold int decode_init(AVCodecContext *avctx)
527 {
528     FMVCContext *s = avctx->priv_data;
529     int i, j, m, block = 0, h = BLOCK_HEIGHT, w = BLOCK_WIDTH;
530
531     switch (avctx->bits_per_coded_sample) {
532     case 16:
533         avctx->pix_fmt = AV_PIX_FMT_RGB555;
534         break;
535     case 24:
536         avctx->pix_fmt = AV_PIX_FMT_BGR24;
537         break;
538     case 32:
539         avctx->pix_fmt = AV_PIX_FMT_BGRA;
540         break;
541     default:
542         av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n",
543                avctx->bits_per_coded_sample);
544         return AVERROR_INVALIDDATA;
545     }
546
547     s->stride = (avctx->width * avctx->bits_per_coded_sample + 31) / 32;
548     s->xb     = s->stride / BLOCK_WIDTH;
549     m         = s->stride % BLOCK_WIDTH;
550     if (m) {
551         if (m < 37) {
552             w = m + BLOCK_WIDTH;
553         } else {
554             w = m;
555             s->xb++;
556         }
557     }
558
559     s->yb = avctx->height / BLOCK_HEIGHT;
560     m     = avctx->height % BLOCK_HEIGHT;
561     if (m) {
562         if (m < 49) {
563             h = m + BLOCK_HEIGHT;
564         } else {
565             h = m;
566             s->yb++;
567         }
568     }
569
570     s->nb_blocks = s->xb * s->yb;
571     if (!s->nb_blocks)
572         return AVERROR_INVALIDDATA;
573     s->blocks    = av_calloc(s->nb_blocks, sizeof(*s->blocks));
574     if (!s->blocks)
575         return AVERROR(ENOMEM);
576
577     for (i = 0; i < s->yb; i++) {
578         for (j = 0; j < s->xb; j++) {
579             if (i != (s->yb - 1) || j != (s->xb - 1)) {
580                 if (i == s->yb - 1) {
581                     s->blocks[block].w    = BLOCK_WIDTH;
582                     s->blocks[block].h    = h;
583                     s->blocks[block].size = BLOCK_WIDTH * h;
584                 } else if (j == s->xb - 1) {
585                     s->blocks[block].w    = w;
586                     s->blocks[block].h    = BLOCK_HEIGHT;
587                     s->blocks[block].size = BLOCK_HEIGHT * w;
588                 } else {
589                     s->blocks[block].w    = BLOCK_WIDTH;
590                     s->blocks[block].h    = BLOCK_HEIGHT;
591                     s->blocks[block].size = BLOCK_WIDTH * BLOCK_HEIGHT;
592                 }
593             } else {
594                 s->blocks[block].w    = w;
595                 s->blocks[block].h    = h;
596                 s->blocks[block].size = w * h;
597             }
598             block++;
599         }
600     }
601
602     s->bpp          = avctx->bits_per_coded_sample >> 3;
603     s->buffer_size  = avctx->width * avctx->height * 4;
604     s->pbuffer_size = avctx->width * avctx->height * 4;
605     s->buffer       = av_mallocz(s->buffer_size);
606     s->pbuffer      = av_mallocz(s->pbuffer_size);
607     if (!s->buffer || !s->pbuffer)
608         return AVERROR(ENOMEM);
609
610     return 0;
611 }
612
613 static av_cold int decode_close(AVCodecContext *avctx)
614 {
615     FMVCContext *s = avctx->priv_data;
616
617     av_freep(&s->buffer);
618     av_freep(&s->pbuffer);
619     av_freep(&s->blocks);
620
621     return 0;
622 }
623
624 AVCodec ff_fmvc_decoder = {
625     .name             = "fmvc",
626     .long_name        = NULL_IF_CONFIG_SMALL("FM Screen Capture Codec"),
627     .type             = AVMEDIA_TYPE_VIDEO,
628     .id               = AV_CODEC_ID_FMVC,
629     .priv_data_size   = sizeof(FMVCContext),
630     .init             = decode_init,
631     .close            = decode_close,
632     .decode           = decode_frame,
633     .capabilities     = AV_CODEC_CAP_DR1,
634     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE |
635                         FF_CODEC_CAP_INIT_CLEANUP,
636 };