speedhq: fix decoding artifacts
[ffmpeg.git] / libavcodec / fmvc.c
1 /*
2  * FM Screen Capture Codec decoder
3  *
4  * Copyright (c) 2017 Paul B Mahol
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <string.h>
26
27 #include "avcodec.h"
28 #include "bytestream.h"
29 #include "internal.h"
30
31 #define BLOCK_HEIGHT 112u
32 #define BLOCK_WIDTH  84u
33
34 typedef struct InterBlock {
35     int      w, h;
36     int      size;
37     int      xor;
38 } InterBlock;
39
40 typedef struct FMVCContext {
41     GetByteContext  gb;
42     PutByteContext  pb;
43     uint8_t        *buffer;
44     size_t          buffer_size;
45     uint8_t        *pbuffer;
46     size_t          pbuffer_size;
47     int             stride;
48     int             bpp;
49     int             yb, xb;
50     InterBlock     *blocks;
51     int             nb_blocks;
52 } FMVCContext;
53
54 static int decode_type2(GetByteContext *gb, PutByteContext *pb)
55 {
56     unsigned repeat = 0, first = 1, opcode;
57     int i, len, pos;
58
59     while (bytestream2_get_bytes_left(gb) > 0) {
60         GetByteContext gbc;
61
62         while (bytestream2_get_bytes_left(gb) > 0) {
63             if (first) {
64                 first = 0;
65                 if (bytestream2_peek_byte(gb) > 17) {
66                     len = bytestream2_get_byte(gb) - 17;
67                     if (len < 4) {
68                         do {
69                             bytestream2_put_byte(pb, bytestream2_get_byte(gb));
70                             --len;
71                         } while (len);
72                         opcode = bytestream2_peek_byte(gb);
73                         continue;
74                     } else {
75                         do {
76                             bytestream2_put_byte(pb, bytestream2_get_byte(gb));
77                             --len;
78                         } while (len);
79                         opcode = bytestream2_peek_byte(gb);
80                         if (opcode < 0x10) {
81                             bytestream2_skip(gb, 1);
82                             pos = - (opcode >> 2) - 4 * bytestream2_get_byte(gb) - 2049;
83
84                             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
85                             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
86
87                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
88                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
89                             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
90                             len = opcode & 3;
91                             if (!len) {
92                                 repeat = 1;
93                             } else {
94                                 do {
95                                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
96                                     --len;
97                                 } while (len);
98                                 opcode = bytestream2_peek_byte(gb);
99                             }
100                             continue;
101                         }
102                     }
103                     repeat = 0;
104                 }
105                 repeat = 1;
106             }
107             if (repeat) {
108                 repeat = 0;
109                 opcode = bytestream2_peek_byte(gb);
110                 if (opcode < 0x10) {
111                     bytestream2_skip(gb, 1);
112                     if (!opcode) {
113                         if (!bytestream2_peek_byte(gb)) {
114                             do {
115                                 bytestream2_skip(gb, 1);
116                                 opcode += 255;
117                             } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
118                         }
119                         opcode += bytestream2_get_byte(gb) + 15;
120                     }
121                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
122                     for (i = opcode - 1; i > 0; --i)
123                         bytestream2_put_byte(pb, bytestream2_get_byte(gb));
124                     opcode = bytestream2_peek_byte(gb);
125                     if (opcode < 0x10) {
126                         bytestream2_skip(gb, 1);
127                         pos = - (opcode >> 2) - 4 * bytestream2_get_byte(gb) - 2049;
128
129                         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
130                         bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
131
132                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
133                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
134                         bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
135                         len = opcode & 3;
136                         if (!len) {
137                             repeat = 1;
138                         } else {
139                             do {
140                                 bytestream2_put_byte(pb, bytestream2_get_byte(gb));
141                                 --len;
142                             } while (len);
143                             opcode = bytestream2_peek_byte(gb);
144                         }
145                         continue;
146                     }
147                 }
148             }
149
150             if (opcode >= 0x40) {
151                 bytestream2_skip(gb, 1);
152                 pos = - ((opcode >> 2) & 7) - 1 - 8 * bytestream2_get_byte(gb);
153                 len = (opcode >> 5) - 1;
154
155                 bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
156                 bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
157
158                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
159                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
160                 do {
161                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
162                     --len;
163                 } while (len);
164
165                 len = opcode & 3;
166
167                 if (!len) {
168                     repeat = 1;
169                 } else {
170                     do {
171                         bytestream2_put_byte(pb, bytestream2_get_byte(gb));
172                         --len;
173                     } while (len);
174                     opcode = bytestream2_peek_byte(gb);
175                 }
176                 continue;
177             } else if (opcode < 0x20) {
178                 break;
179             }
180             len = opcode & 0x1F;
181             bytestream2_skip(gb, 1);
182             if (!len) {
183                 if (!bytestream2_peek_byte(gb)) {
184                     do {
185                         bytestream2_skip(gb, 1);
186                         len += 255;
187                     } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
188                 }
189                 len += bytestream2_get_byte(gb) + 31;
190             }
191             i = bytestream2_get_le16(gb);
192             pos = - (i >> 2) - 1;
193
194             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
195             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
196
197             if (len < 6 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
198                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
199                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
200                 do {
201                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
202                     --len;
203                 } while (len);
204             } else {
205                 bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
206                 for (len = len - 2; len; --len)
207                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
208             }
209             len = i & 3;
210             if (!len) {
211                 repeat = 1;
212             } else {
213                 do {
214                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
215                     --len;
216                 } while (len);
217                 opcode = bytestream2_peek_byte(gb);
218             }
219         }
220         bytestream2_skip(gb, 1);
221         if (opcode < 0x10) {
222             pos = -(opcode >> 2) - 1 - 4 * bytestream2_get_byte(gb);
223
224             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
225             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
226
227             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
228             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
229             len = opcode & 3;
230             if (!len) {
231                 repeat = 1;
232             } else {
233                 do {
234                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
235                     --len;
236                 } while (len);
237                 opcode = bytestream2_peek_byte(gb);
238             }
239             continue;
240         }
241         len = opcode & 7;
242         if (!len) {
243             if (!bytestream2_peek_byte(gb)) {
244                 do {
245                     bytestream2_skip(gb, 1);
246                     len += 255;
247                 } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
248             }
249             len += bytestream2_get_byte(gb) + 7;
250         }
251         i = bytestream2_get_le16(gb);
252         pos = bytestream2_tell_p(pb) - 2048 * (opcode & 8);
253         pos = pos - (i >> 2);
254         if (pos == bytestream2_tell_p(pb))
255             break;
256
257         pos = pos - 0x4000;
258         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
259         bytestream2_seek(&gbc, pos, SEEK_SET);
260
261         if (len < 6 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
262             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
263             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
264             do {
265                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
266                 --len;
267             } while (len);
268         } else {
269             bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
270             for (len = len - 2; len; --len)
271                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
272         }
273
274         len = i & 3;
275         if (!len) {
276             repeat = 1;
277         } else {
278             do {
279                 bytestream2_put_byte(pb, bytestream2_get_byte(gb));
280                 --len;
281             } while (len);
282             opcode = bytestream2_peek_byte(gb);
283         }
284     }
285
286     return 0;
287 }
288
289 static int decode_type1(GetByteContext *gb, PutByteContext *pb)
290 {
291     unsigned opcode, len;
292     int high = 0;
293     int i, pos;
294
295     while (bytestream2_get_bytes_left(gb) > 0) {
296         GetByteContext gbc;
297
298         while (bytestream2_get_bytes_left(gb) > 0) {
299             while (bytestream2_get_bytes_left(gb) > 0) {
300                 opcode = bytestream2_get_byte(gb);
301                 high = opcode >= 0x20;
302                 if (high)
303                     break;
304                 if (opcode)
305                     break;
306                 opcode = bytestream2_get_byte(gb);
307                 if (opcode < 0xF8) {
308                     opcode = opcode + 32;
309                     break;
310                 }
311                 i = opcode - 0xF8;
312                 if (i) {
313                     len = 256;
314                     do {
315                         len *= 2;
316                         --i;
317                     } while (i);
318                 } else {
319                     len = 280;
320                 }
321                 do {
322                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
323                     bytestream2_put_le32(pb, bytestream2_get_le32(gb));
324                     len -= 8;
325                 } while (len && bytestream2_get_bytes_left(gb) > 0);
326             }
327
328             if (!high) {
329                 do {
330                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
331                     --opcode;
332                 } while (opcode && bytestream2_get_bytes_left(gb) > 0);
333
334                 while (bytestream2_get_bytes_left(gb) > 0) {
335                     GetByteContext gbc;
336
337                     opcode = bytestream2_get_byte(gb);
338                     if (opcode >= 0x20)
339                         break;
340                     bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
341
342                     pos = -(opcode | 32 * bytestream2_get_byte(gb)) - 1;
343                     bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
344                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
345                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
346                     bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
347                     bytestream2_put_byte(pb, bytestream2_get_byte(gb));
348                 }
349             }
350             high = 0;
351             if (opcode < 0x40)
352                 break;
353             bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
354             pos = (-((opcode & 0x1F) | 32 * bytestream2_get_byte(gb)) - 1);
355             bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos, SEEK_SET);
356             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
357             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
358             len = (opcode >> 5) - 1;
359             do {
360                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
361                 --len;
362             } while (len && bytestream2_get_bytes_left(&gbc) > 0);
363         }
364         len = opcode & 0x1F;
365         if (!len) {
366             if (!bytestream2_peek_byte(gb)) {
367                 do {
368                     bytestream2_skip(gb, 1);
369                     len += 255;
370                 } while (!bytestream2_peek_byte(gb) && bytestream2_get_bytes_left(gb) > 0);
371             }
372             len += bytestream2_get_byte(gb) + 31;
373         }
374         pos = -bytestream2_get_byte(gb);
375         bytestream2_init(&gbc, pb->buffer_start, pb->buffer_end - pb->buffer_start);
376         bytestream2_seek(&gbc, bytestream2_tell_p(pb) + pos - (bytestream2_get_byte(gb) << 8), SEEK_SET);
377         if (bytestream2_tell_p(pb) == bytestream2_tell(&gbc))
378             break;
379         if (len < 5 || bytestream2_tell_p(pb) - bytestream2_tell(&gbc) < 4) {
380             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
381             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
382             bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
383             do {
384                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
385                 --len;
386             } while (len && bytestream2_get_bytes_left(&gbc) > 0);
387         } else {
388             bytestream2_put_le32(pb, bytestream2_get_le32(&gbc));
389             len--;
390             do {
391                 bytestream2_put_byte(pb, bytestream2_get_byte(&gbc));
392                 len--;
393             } while (len && bytestream2_get_bytes_left(&gbc) > 0);
394         }
395     }
396
397     return 0;
398 }
399
400 static int decode_frame(AVCodecContext *avctx,
401                         void *data, int *got_frame,
402                         AVPacket *avpkt)
403 {
404     FMVCContext *s = avctx->priv_data;
405     GetByteContext *gb = &s->gb;
406     PutByteContext *pb = &s->pb;
407     AVFrame *frame = data;
408     int ret, y, x;
409
410     if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
411         return ret;
412
413     bytestream2_init(gb, avpkt->data, avpkt->size);
414     bytestream2_skip(gb, 2);
415
416     frame->key_frame = !!bytestream2_get_le16(gb);
417     frame->pict_type = frame->key_frame ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_P;
418
419     if (frame->key_frame) {
420         const uint8_t *src;
421         int type, size;
422         uint8_t *dst;
423
424         type = bytestream2_get_le16(gb);
425         size = bytestream2_get_le16(gb);
426         if (size > bytestream2_get_bytes_left(gb))
427             return AVERROR_INVALIDDATA;
428
429         bytestream2_init_writer(pb, s->buffer, s->buffer_size);
430         if (type == 1) {
431             decode_type1(gb, pb);
432         } else if (type == 2){
433             decode_type2(gb, pb);
434         } else {
435             avpriv_report_missing_feature(avctx, "compression %d", type);
436             return AVERROR_PATCHWELCOME;
437         }
438
439         src = s->buffer;
440         dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
441         for (y = 0; y < avctx->height; y++) {
442             memcpy(dst, src, avctx->width * s->bpp);
443             dst -= frame->linesize[0];
444             src += s->stride * 4;
445         }
446     } else {
447         int block, nb_blocks, type, k, l;
448         uint8_t *ssrc, *ddst;
449         const uint32_t *src;
450         uint32_t *dst;
451
452         for (block = 0; block < s->nb_blocks; block++)
453             s->blocks[block].xor = 0;
454
455         nb_blocks = bytestream2_get_le16(gb);
456         if (nb_blocks > s->nb_blocks)
457             return AVERROR_INVALIDDATA;
458
459         bytestream2_init_writer(pb, s->pbuffer, s->pbuffer_size);
460
461         type = bytestream2_get_le16(gb);
462         for (block = 0; block < nb_blocks; block++) {
463             int size, offset, start = 0;
464
465             offset = bytestream2_get_le16(gb);
466             if (offset > s->nb_blocks)
467                 return AVERROR_INVALIDDATA;
468
469             size = bytestream2_get_le16(gb);
470             if (size > bytestream2_get_bytes_left(gb))
471                 return AVERROR_INVALIDDATA;
472
473             start = bytestream2_tell_p(pb);
474             if (type == 1) {
475                 decode_type1(gb, pb);
476             } else if (type == 2){
477                 decode_type2(gb, pb);
478             } else {
479                 avpriv_report_missing_feature(avctx, "compression %d", type);
480                 return AVERROR_PATCHWELCOME;
481             }
482
483             if (s->blocks[offset].size * 4 != bytestream2_tell_p(pb) - start)
484                 return AVERROR_INVALIDDATA;
485
486             s->blocks[offset].xor = 1;
487         }
488
489         src = (const uint32_t *)s->pbuffer;
490         dst = (uint32_t *)s->buffer;
491
492         for (block = 0, y = 0; y < s->yb; y++) {
493             int block_h = s->blocks[block].h;
494             uint32_t *rect = dst;
495
496             for (x = 0; x < s->xb; x++) {
497                 int block_w = s->blocks[block].w;
498                 uint32_t *row = dst;
499
500                 block_h = s->blocks[block].h;
501                 if (s->blocks[block].xor) {
502                     for (k = 0; k < block_h; k++) {
503                         uint32_t *column = dst;
504                         for (l = 0; l < block_w; l++) {
505                             *dst++ ^= *src++;
506                         }
507                         dst = &column[s->stride];
508                     }
509                 }
510                 dst = &row[block_w];
511                 ++block;
512             }
513             dst = &rect[block_h * s->stride];
514         }
515
516         ssrc = s->buffer;
517         ddst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
518         for (y = 0; y < avctx->height; y++) {
519             memcpy(ddst, ssrc, avctx->width * s->bpp);
520             ddst -= frame->linesize[0];
521             ssrc += s->stride * 4;
522         }
523     }
524
525     *got_frame = 1;
526
527     return avpkt->size;
528 }
529
530 static av_cold int decode_init(AVCodecContext *avctx)
531 {
532     FMVCContext *s = avctx->priv_data;
533     int i, j, m, block = 0, h = BLOCK_HEIGHT, w = BLOCK_WIDTH;
534
535     switch (avctx->bits_per_coded_sample) {
536     case 16: avctx->pix_fmt = AV_PIX_FMT_RGB555; break;
537     case 24: avctx->pix_fmt = AV_PIX_FMT_BGR24;  break;
538     case 32: avctx->pix_fmt = AV_PIX_FMT_BGRA;   break;
539     default:
540         av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", avctx->bits_per_coded_sample);
541         return AVERROR_INVALIDDATA;
542     }
543
544     s->stride = (avctx->width * avctx->bits_per_coded_sample + 31) / 32;
545     s->xb = s->stride / BLOCK_WIDTH;
546     m = s->stride % BLOCK_WIDTH;
547     if (m) {
548         if (m < 37) {
549             w = m + BLOCK_WIDTH;
550         } else {
551             w = m;
552             s->xb++;
553         }
554     }
555
556     s->yb = avctx->height / BLOCK_HEIGHT;
557     m = avctx->height % BLOCK_HEIGHT;
558     if (m) {
559         if (m < 49) {
560             h = m + BLOCK_HEIGHT;
561         } else {
562             h = m;
563             s->yb++;
564         }
565     }
566
567     s->nb_blocks = s->xb * s->yb;
568     s->blocks = av_calloc(s->nb_blocks, sizeof(*s->blocks));
569     if (!s->blocks)
570         return AVERROR(ENOMEM);
571
572     for (i = 0; i < s->yb; i++) {
573         for (j = 0; j < s->xb; j++) {
574             if (i != (s->yb - 1) || j != (s->xb - 1)) {
575                 if (i == s->yb - 1) {
576                     s->blocks[block].w = BLOCK_WIDTH;
577                     s->blocks[block].h = h;
578                     s->blocks[block].size = BLOCK_WIDTH * h;
579                 } else if (j == s->xb - 1) {
580                     s->blocks[block].w = w;
581                     s->blocks[block].h = BLOCK_HEIGHT;
582                     s->blocks[block].size = BLOCK_HEIGHT * w;
583                 } else {
584                     s->blocks[block].w = BLOCK_WIDTH;
585                     s->blocks[block].h = BLOCK_HEIGHT;
586                     s->blocks[block].size = BLOCK_WIDTH * BLOCK_HEIGHT;
587                 }
588             } else {
589                 s->blocks[block].w = w;
590                 s->blocks[block].h = h;
591                 s->blocks[block].size = w * h;
592             }
593             block++;
594         }
595     }
596
597     s->bpp = avctx->bits_per_coded_sample >> 3;
598     s->buffer_size = avctx->width * avctx->height * 4;
599     s->pbuffer_size = avctx->width * avctx->height * 4;
600     s->buffer = av_malloc(s->buffer_size);
601     s->pbuffer = av_malloc(s->pbuffer_size);
602     if (!s->buffer || !s->pbuffer)
603         return AVERROR(ENOMEM);
604
605     return 0;
606 }
607
608 static av_cold int decode_close(AVCodecContext *avctx)
609 {
610     FMVCContext *s = avctx->priv_data;
611
612     av_freep(&s->buffer);
613     av_freep(&s->pbuffer);
614     av_freep(&s->blocks);
615
616     return 0;
617 }
618
619 AVCodec ff_fmvc_decoder = {
620     .name             = "fmvc",
621     .long_name        = NULL_IF_CONFIG_SMALL("FM Screen Capture Codec"),
622     .type             = AVMEDIA_TYPE_VIDEO,
623     .id               = AV_CODEC_ID_FMVC,
624     .priv_data_size   = sizeof(FMVCContext),
625     .init             = decode_init,
626     .close            = decode_close,
627     .decode           = decode_frame,
628     .capabilities     = AV_CODEC_CAP_DR1,
629     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE |
630                         FF_CODEC_CAP_INIT_CLEANUP,
631 };