avcodec/j2kenc: YUV should not use the ICT
[ffmpeg.git] / libavcodec / j2kenc.c
1 /*
2  * JPEG2000 image encoder
3  * Copyright (c) 2007 Kamil Nowosad
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * JPEG2000 image encoder
24  * @file
25  * @author Kamil Nowosad
26  */
27
28 #include <float.h>
29 #include "avcodec.h"
30 #include "internal.h"
31 #include "bytestream.h"
32 #include "jpeg2000.h"
33 #include "libavutil/common.h"
34
35 #define NMSEDEC_BITS 7
36 #define NMSEDEC_FRACBITS (NMSEDEC_BITS-1)
37 #define WMSEDEC_SHIFT 13 ///< must be >= 13
38 #define LAMBDA_SCALE (100000000LL << (WMSEDEC_SHIFT - 13))
39
40 static int lut_nmsedec_ref [1<<NMSEDEC_BITS],
41            lut_nmsedec_ref0[1<<NMSEDEC_BITS],
42            lut_nmsedec_sig [1<<NMSEDEC_BITS],
43            lut_nmsedec_sig0[1<<NMSEDEC_BITS];
44
45 static const int dwt_norms[2][4][10] = { // [dwt_type][band][rlevel] (multiplied by 10000)
46     {{10000, 19650, 41770,  84030, 169000, 338400,  676900, 1353000, 2706000, 5409000},
47      {20220, 39890, 83550, 170400, 342700, 686300, 1373000, 2746000, 5490000},
48      {20220, 39890, 83550, 170400, 342700, 686300, 1373000, 2746000, 5490000},
49      {20800, 38650, 83070, 171800, 347100, 695900, 1393000, 2786000, 5572000}},
50
51     {{10000, 15000, 27500, 53750, 106800, 213400, 426700, 853300, 1707000, 3413000},
52      {10380, 15920, 29190, 57030, 113300, 226400, 452500, 904800, 1809000},
53      {10380, 15920, 29190, 57030, 113300, 226400, 452500, 904800, 1809000},
54      { 7186,  9218, 15860, 30430,  60190, 120100, 240000, 479700,  959300}}
55 };
56
57 typedef struct {
58    Jpeg2000Component *comp;
59 } Jpeg2000Tile;
60
61 typedef struct {
62     AVCodecContext *avctx;
63     const AVFrame *picture;
64
65     int width, height; ///< image width and height
66     uint8_t cbps[4]; ///< bits per sample in particular components
67     int chroma_shift[2];
68     uint8_t planar;
69     int ncomponents;
70     int tile_width, tile_height; ///< tile size
71     int numXtiles, numYtiles;
72
73     uint8_t *buf_start;
74     uint8_t *buf;
75     uint8_t *buf_end;
76     int bit_index;
77
78     int64_t lambda;
79
80     Jpeg2000CodingStyle codsty;
81     Jpeg2000QuantStyle  qntsty;
82
83     Jpeg2000Tile *tile;
84 } Jpeg2000EncoderContext;
85
86
87 /* debug */
88 #if 0
89 #undef ifprintf
90 #undef printf
91
92 static void nspaces(FILE *fd, int n)
93 {
94     while(n--) putc(' ', fd);
95 }
96
97 static void printcomp(Jpeg2000Component *comp)
98 {
99     int i;
100     for (i = 0; i < comp->y1 - comp->y0; i++)
101         ff_jpeg2000_printv(comp->i_data + i * (comp->x1 - comp->x0), comp->x1 - comp->x0);
102 }
103
104 static void dump(Jpeg2000EncoderContext *s, FILE *fd)
105 {
106     int tileno, compno, reslevelno, bandno, precno;
107     fprintf(fd, "XSiz = %d, YSiz = %d, tile_width = %d, tile_height = %d\n"
108                 "numXtiles = %d, numYtiles = %d, ncomponents = %d\n"
109                 "tiles:\n",
110             s->width, s->height, s->tile_width, s->tile_height,
111             s->numXtiles, s->numYtiles, s->ncomponents);
112     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
113         Jpeg2000Tile *tile = s->tile + tileno;
114         nspaces(fd, 2);
115         fprintf(fd, "tile %d:\n", tileno);
116         for(compno = 0; compno < s->ncomponents; compno++){
117             Jpeg2000Component *comp = tile->comp + compno;
118             nspaces(fd, 4);
119             fprintf(fd, "component %d:\n", compno);
120             nspaces(fd, 4);
121             fprintf(fd, "x0 = %d, x1 = %d, y0 = %d, y1 = %d\n",
122                         comp->x0, comp->x1, comp->y0, comp->y1);
123             for(reslevelno = 0; reslevelno < s->nreslevels; reslevelno++){
124                 Jpeg2000ResLevel *reslevel = comp->reslevel + reslevelno;
125                 nspaces(fd, 6);
126                 fprintf(fd, "reslevel %d:\n", reslevelno);
127                 nspaces(fd, 6);
128                 fprintf(fd, "x0 = %d, x1 = %d, y0 = %d, y1 = %d, nbands = %d\n",
129                         reslevel->x0, reslevel->x1, reslevel->y0,
130                         reslevel->y1, reslevel->nbands);
131                 for(bandno = 0; bandno < reslevel->nbands; bandno++){
132                     Jpeg2000Band *band = reslevel->band + bandno;
133                     nspaces(fd, 8);
134                     fprintf(fd, "band %d:\n", bandno);
135                     nspaces(fd, 8);
136                     fprintf(fd, "x0 = %d, x1 = %d, y0 = %d, y1 = %d,"
137                                 "codeblock_width = %d, codeblock_height = %d cblknx = %d cblkny = %d\n",
138                                 band->x0, band->x1,
139                                 band->y0, band->y1,
140                                 band->codeblock_width, band->codeblock_height,
141                                 band->cblknx, band->cblkny);
142                     for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++){
143                         Jpeg2000Prec *prec = band->prec + precno;
144                         nspaces(fd, 10);
145                         fprintf(fd, "prec %d:\n", precno);
146                         nspaces(fd, 10);
147                         fprintf(fd, "xi0 = %d, xi1 = %d, yi0 = %d, yi1 = %d\n",
148                                      prec->xi0, prec->xi1, prec->yi0, prec->yi1);
149                     }
150                 }
151             }
152         }
153     }
154 }
155 #endif
156
157 /* bitstream routines */
158
159 /** put n times val bit */
160 static void put_bits(Jpeg2000EncoderContext *s, int val, int n) // TODO: optimize
161 {
162     while (n-- > 0){
163         if (s->bit_index == 8)
164         {
165             s->bit_index = *s->buf == 0xff;
166             *(++s->buf) = 0;
167         }
168         *s->buf |= val << (7 - s->bit_index++);
169     }
170 }
171
172 /** put n least significant bits of a number num */
173 static void put_num(Jpeg2000EncoderContext *s, int num, int n)
174 {
175     while(--n >= 0)
176         put_bits(s, (num >> n) & 1, 1);
177 }
178
179 /** flush the bitstream */
180 static void j2k_flush(Jpeg2000EncoderContext *s)
181 {
182     if (s->bit_index){
183         s->bit_index = 0;
184         s->buf++;
185     }
186 }
187
188 /* tag tree routines */
189
190 /** code the value stored in node */
191 static void tag_tree_code(Jpeg2000EncoderContext *s, Jpeg2000TgtNode *node, int threshold)
192 {
193     Jpeg2000TgtNode *stack[30];
194     int sp = 1, curval = 0;
195     stack[0] = node;
196
197     node = node->parent;
198     while(node){
199         if (node->vis){
200             curval = node->val;
201             break;
202         }
203         node->vis++;
204         stack[sp++] = node;
205         node = node->parent;
206     }
207     while(--sp >= 0){
208         if (stack[sp]->val >= threshold){
209             put_bits(s, 0, threshold - curval);
210             break;
211         }
212         put_bits(s, 0, stack[sp]->val - curval);
213         put_bits(s, 1, 1);
214         curval = stack[sp]->val;
215     }
216 }
217
218 /** update the value in node */
219 static void tag_tree_update(Jpeg2000TgtNode *node)
220 {
221     int lev = 0;
222     while (node->parent){
223         if (node->parent->val <= node->val)
224             break;
225         node->parent->val = node->val;
226         node = node->parent;
227         lev++;
228     }
229 }
230
231 static int put_siz(Jpeg2000EncoderContext *s)
232 {
233     int i;
234
235     if (s->buf_end - s->buf < 40 + 3 * s->ncomponents)
236         return -1;
237
238     bytestream_put_be16(&s->buf, JPEG2000_SIZ);
239     bytestream_put_be16(&s->buf, 38 + 3 * s->ncomponents); // Lsiz
240     bytestream_put_be16(&s->buf, 0); // Rsiz
241     bytestream_put_be32(&s->buf, s->width); // width
242     bytestream_put_be32(&s->buf, s->height); // height
243     bytestream_put_be32(&s->buf, 0); // X0Siz
244     bytestream_put_be32(&s->buf, 0); // Y0Siz
245
246     bytestream_put_be32(&s->buf, s->tile_width); // XTSiz
247     bytestream_put_be32(&s->buf, s->tile_height); // YTSiz
248     bytestream_put_be32(&s->buf, 0); // XT0Siz
249     bytestream_put_be32(&s->buf, 0); // YT0Siz
250     bytestream_put_be16(&s->buf, s->ncomponents); // CSiz
251
252     for (i = 0; i < s->ncomponents; i++){ // Ssiz_i XRsiz_i, YRsiz_i
253         bytestream_put_byte(&s->buf, 7);
254         bytestream_put_byte(&s->buf, i?1<<s->chroma_shift[0]:1);
255         bytestream_put_byte(&s->buf, i?1<<s->chroma_shift[1]:1);
256     }
257     return 0;
258 }
259
260 static int put_cod(Jpeg2000EncoderContext *s)
261 {
262     Jpeg2000CodingStyle *codsty = &s->codsty;
263
264     if (s->buf_end - s->buf < 14)
265         return -1;
266
267     bytestream_put_be16(&s->buf, JPEG2000_COD);
268     bytestream_put_be16(&s->buf, 12); // Lcod
269     bytestream_put_byte(&s->buf, 0);  // Scod
270     // SGcod
271     bytestream_put_byte(&s->buf, 0); // progression level
272     bytestream_put_be16(&s->buf, 1); // num of layers
273     if(s->avctx->pix_fmt == AV_PIX_FMT_YUV444P){
274         bytestream_put_byte(&s->buf, 0); // unspecified
275     }else{
276         bytestream_put_byte(&s->buf, 0); // unspecified
277     }
278     // SPcod
279     bytestream_put_byte(&s->buf, codsty->nreslevels - 1); // num of decomp. levels
280     bytestream_put_byte(&s->buf, codsty->log2_cblk_width-2); // cblk width
281     bytestream_put_byte(&s->buf, codsty->log2_cblk_height-2); // cblk height
282     bytestream_put_byte(&s->buf, 0); // cblk style
283     bytestream_put_byte(&s->buf, codsty->transform == FF_DWT53); // transformation
284     return 0;
285 }
286
287 static int put_qcd(Jpeg2000EncoderContext *s, int compno)
288 {
289     int i, size;
290     Jpeg2000CodingStyle *codsty = &s->codsty;
291     Jpeg2000QuantStyle  *qntsty = &s->qntsty;
292
293     if (qntsty->quantsty == JPEG2000_QSTY_NONE)
294         size = 4 + 3 * (codsty->nreslevels-1);
295     else // QSTY_SE
296         size = 5 + 6 * (codsty->nreslevels-1);
297
298     if (s->buf_end - s->buf < size + 2)
299         return -1;
300
301     bytestream_put_be16(&s->buf, JPEG2000_QCD);
302     bytestream_put_be16(&s->buf, size);  // LQcd
303     bytestream_put_byte(&s->buf, (qntsty->nguardbits << 5) | qntsty->quantsty);  // Sqcd
304     if (qntsty->quantsty == JPEG2000_QSTY_NONE)
305         for (i = 0; i < codsty->nreslevels * 3 - 2; i++)
306             bytestream_put_byte(&s->buf, qntsty->expn[i] << 3);
307     else // QSTY_SE
308         for (i = 0; i < codsty->nreslevels * 3 - 2; i++)
309             bytestream_put_be16(&s->buf, (qntsty->expn[i] << 11) | qntsty->mant[i]);
310     return 0;
311 }
312
313 static uint8_t *put_sot(Jpeg2000EncoderContext *s, int tileno)
314 {
315     uint8_t *psotptr;
316
317     if (s->buf_end - s->buf < 12)
318         return NULL;
319
320     bytestream_put_be16(&s->buf, JPEG2000_SOT);
321     bytestream_put_be16(&s->buf, 10); // Lsot
322     bytestream_put_be16(&s->buf, tileno); // Isot
323
324     psotptr = s->buf;
325     bytestream_put_be32(&s->buf, 0); // Psot (filled in later)
326
327     bytestream_put_byte(&s->buf, 0); // TPsot
328     bytestream_put_byte(&s->buf, 1); // TNsot
329     return psotptr;
330 }
331
332 /**
333  * compute the sizes of tiles, resolution levels, bands, etc.
334  * allocate memory for them
335  * divide the input image into tile-components
336  */
337 static int init_tiles(Jpeg2000EncoderContext *s)
338 {
339     int tileno, tilex, tiley, compno;
340     Jpeg2000CodingStyle *codsty = &s->codsty;
341     Jpeg2000QuantStyle  *qntsty = &s->qntsty;
342
343     s->numXtiles = ff_jpeg2000_ceildiv(s->width, s->tile_width);
344     s->numYtiles = ff_jpeg2000_ceildiv(s->height, s->tile_height);
345
346     s->tile = av_malloc_array(s->numXtiles, s->numYtiles * sizeof(Jpeg2000Tile));
347     if (!s->tile)
348         return AVERROR(ENOMEM);
349     for (tileno = 0, tiley = 0; tiley < s->numYtiles; tiley++)
350         for (tilex = 0; tilex < s->numXtiles; tilex++, tileno++){
351             Jpeg2000Tile *tile = s->tile + tileno;
352
353             tile->comp = av_mallocz_array(s->ncomponents, sizeof(Jpeg2000Component));
354             if (!tile->comp)
355                 return AVERROR(ENOMEM);
356             for (compno = 0; compno < s->ncomponents; compno++){
357                 Jpeg2000Component *comp = tile->comp + compno;
358                 int ret, i, j;
359
360                 comp->coord[0][0] = comp->coord_o[0][0] = tilex * s->tile_width;
361                 comp->coord[0][1] = comp->coord_o[0][1] = FFMIN((tilex+1)*s->tile_width, s->width);
362                 comp->coord[1][0] = comp->coord_o[1][0] = tiley * s->tile_height;
363                 comp->coord[1][1] = comp->coord_o[1][1] = FFMIN((tiley+1)*s->tile_height, s->height);
364                 if (compno > 0)
365                     for (i = 0; i < 2; i++)
366                         for (j = 0; j < 2; j++)
367                             comp->coord[i][j] = comp->coord_o[i][j] = ff_jpeg2000_ceildivpow2(comp->coord[i][j], s->chroma_shift[i]);
368
369                 if ((ret = ff_jpeg2000_init_component(comp,
370                                                 codsty,
371                                                 qntsty,
372                                                 s->cbps[compno],
373                                                 compno?1<<s->chroma_shift[0]:1,
374                                                 compno?1<<s->chroma_shift[1]:1,
375                                                 s->avctx
376                                                )) < 0)
377                     return ret;
378             }
379         }
380     return 0;
381 }
382
383 static void copy_frame(Jpeg2000EncoderContext *s)
384 {
385     int tileno, compno, i, y, x;
386     uint8_t *line;
387     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
388         Jpeg2000Tile *tile = s->tile + tileno;
389         if (s->planar){
390             for (compno = 0; compno < s->ncomponents; compno++){
391                 Jpeg2000Component *comp = tile->comp + compno;
392                 int *dst = comp->i_data;
393                 line = s->picture->data[compno]
394                        + comp->coord[1][0] * s->picture->linesize[compno]
395                        + comp->coord[0][0];
396                 for (y = comp->coord[1][0]; y < comp->coord[1][1]; y++){
397                     uint8_t *ptr = line;
398                     for (x = comp->coord[0][0]; x < comp->coord[0][1]; x++)
399                         *dst++ = *ptr++ - (1 << 7);
400                     line += s->picture->linesize[compno];
401                 }
402             }
403         } else{
404             line = s->picture->data[0] + tile->comp[0].coord[1][0] * s->picture->linesize[0]
405                    + tile->comp[0].coord[0][0] * s->ncomponents;
406
407             i = 0;
408             for (y = tile->comp[0].coord[1][0]; y < tile->comp[0].coord[1][1]; y++){
409                 uint8_t *ptr = line;
410                 for (x = tile->comp[0].coord[0][0]; x < tile->comp[0].coord[0][1]; x++, i++){
411                     for (compno = 0; compno < s->ncomponents; compno++){
412                         tile->comp[compno].i_data[i] = *ptr++  - (1 << 7);
413                     }
414                 }
415                 line += s->picture->linesize[0];
416             }
417         }
418     }
419 }
420
421 static void init_quantization(Jpeg2000EncoderContext *s)
422 {
423     int compno, reslevelno, bandno;
424     Jpeg2000QuantStyle  *qntsty = &s->qntsty;
425     Jpeg2000CodingStyle *codsty = &s->codsty;
426
427     for (compno = 0; compno < s->ncomponents; compno++){
428         int gbandno = 0;
429         for (reslevelno = 0; reslevelno < codsty->nreslevels; reslevelno++){
430             int nbands, lev = codsty->nreslevels - reslevelno - 1;
431             nbands = reslevelno ? 3 : 1;
432             for (bandno = 0; bandno < nbands; bandno++, gbandno++){
433                 int expn, mant = 0;
434
435                 if (codsty->transform == FF_DWT97_INT){
436                     int bandpos = bandno + (reslevelno>0),
437                         ss = 81920000 / dwt_norms[0][bandpos][lev],
438                         log = av_log2(ss);
439                     mant = (11 - log < 0 ? ss >> log - 11 : ss << 11 - log) & 0x7ff;
440                     expn = s->cbps[compno] - log + 13;
441                 } else
442                     expn = ((bandno&2)>>1) + (reslevelno>0) + s->cbps[compno];
443
444                 qntsty->expn[gbandno] = expn;
445                 qntsty->mant[gbandno] = mant;
446             }
447         }
448     }
449 }
450
451 static void init_luts(void)
452 {
453     int i, a,
454         mask = ~((1<<NMSEDEC_FRACBITS)-1);
455
456     for (i = 0; i < (1 << NMSEDEC_BITS); i++){
457         lut_nmsedec_sig[i]  = FFMAX(6*i - (9<<NMSEDEC_FRACBITS-1) << 12-NMSEDEC_FRACBITS, 0);
458         lut_nmsedec_sig0[i] = FFMAX((i*i + (1<<NMSEDEC_FRACBITS-1) & mask) << 1, 0);
459
460         a = (i >> (NMSEDEC_BITS-2)&2) + 1;
461         lut_nmsedec_ref[i]  = FFMAX((-2*i + (1<<NMSEDEC_FRACBITS) + a*i - (a*a<<NMSEDEC_FRACBITS-2))
462                                     << 13-NMSEDEC_FRACBITS, 0);
463         lut_nmsedec_ref0[i] = FFMAX(((i*i + (1-4*i << NMSEDEC_FRACBITS-1) + (1<<2*NMSEDEC_FRACBITS)) & mask)
464                                     << 1, 0);
465     }
466 }
467
468 /* tier-1 routines */
469 static int getnmsedec_sig(int x, int bpno)
470 {
471     if (bpno > NMSEDEC_FRACBITS)
472         return lut_nmsedec_sig[(x >> (bpno - NMSEDEC_FRACBITS)) & ((1 << NMSEDEC_BITS) - 1)];
473     return lut_nmsedec_sig0[x & ((1 << NMSEDEC_BITS) - 1)];
474 }
475
476 static int getnmsedec_ref(int x, int bpno)
477 {
478     if (bpno > NMSEDEC_FRACBITS)
479         return lut_nmsedec_ref[(x >> (bpno - NMSEDEC_FRACBITS)) & ((1 << NMSEDEC_BITS) - 1)];
480     return lut_nmsedec_ref0[x & ((1 << NMSEDEC_BITS) - 1)];
481 }
482
483 static void encode_sigpass(Jpeg2000T1Context *t1, int width, int height, int bandno, int *nmsedec, int bpno)
484 {
485     int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
486     for (y0 = 0; y0 < height; y0 += 4)
487         for (x = 0; x < width; x++)
488             for (y = y0; y < height && y < y0+4; y++){
489                 if (!(t1->flags[y+1][x+1] & JPEG2000_T1_SIG) && (t1->flags[y+1][x+1] & JPEG2000_T1_SIG_NB)){
490                     int ctxno = ff_jpeg2000_getsigctxno(t1->flags[y+1][x+1], bandno),
491                         bit = t1->data[y][x] & mask ? 1 : 0;
492                     ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, bit);
493                     if (bit){
494                         int xorbit;
495                         int ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
496                         ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, (t1->flags[y+1][x+1] >> 15) ^ xorbit);
497                         *nmsedec += getnmsedec_sig(t1->data[y][x], bpno + NMSEDEC_FRACBITS);
498                         ff_jpeg2000_set_significance(t1, x, y, t1->flags[y+1][x+1] >> 15);
499                     }
500                     t1->flags[y+1][x+1] |= JPEG2000_T1_VIS;
501                 }
502             }
503 }
504
505 static void encode_refpass(Jpeg2000T1Context *t1, int width, int height, int *nmsedec, int bpno)
506 {
507     int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
508     for (y0 = 0; y0 < height; y0 += 4)
509         for (x = 0; x < width; x++)
510             for (y = y0; y < height && y < y0+4; y++)
511                 if ((t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS)) == JPEG2000_T1_SIG){
512                     int ctxno = ff_jpeg2000_getrefctxno(t1->flags[y+1][x+1]);
513                     *nmsedec += getnmsedec_ref(t1->data[y][x], bpno + NMSEDEC_FRACBITS);
514                     ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, t1->data[y][x] & mask ? 1:0);
515                     t1->flags[y+1][x+1] |= JPEG2000_T1_REF;
516                 }
517 }
518
519 static void encode_clnpass(Jpeg2000T1Context *t1, int width, int height, int bandno, int *nmsedec, int bpno)
520 {
521     int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
522     for (y0 = 0; y0 < height; y0 += 4)
523         for (x = 0; x < width; x++){
524             if (y0 + 3 < height && !(
525             (t1->flags[y0+1][x+1] & (JPEG2000_T1_SIG_NB | JPEG2000_T1_VIS | JPEG2000_T1_SIG)) ||
526             (t1->flags[y0+2][x+1] & (JPEG2000_T1_SIG_NB | JPEG2000_T1_VIS | JPEG2000_T1_SIG)) ||
527             (t1->flags[y0+3][x+1] & (JPEG2000_T1_SIG_NB | JPEG2000_T1_VIS | JPEG2000_T1_SIG)) ||
528             (t1->flags[y0+4][x+1] & (JPEG2000_T1_SIG_NB | JPEG2000_T1_VIS | JPEG2000_T1_SIG))))
529             {
530                 // aggregation mode
531                 int rlen;
532                 for (rlen = 0; rlen < 4; rlen++)
533                     if (t1->data[y0+rlen][x] & mask)
534                         break;
535                 ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + MQC_CX_RL, rlen != 4);
536                 if (rlen == 4)
537                     continue;
538                 ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + MQC_CX_UNI, rlen >> 1);
539                 ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + MQC_CX_UNI, rlen & 1);
540                 for (y = y0 + rlen; y < y0 + 4; y++){
541                     if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))){
542                         int ctxno = ff_jpeg2000_getsigctxno(t1->flags[y+1][x+1], bandno);
543                         if (y > y0 + rlen)
544                             ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, t1->data[y][x] & mask ? 1:0);
545                         if (t1->data[y][x] & mask){ // newly significant
546                             int xorbit;
547                             int ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
548                             *nmsedec += getnmsedec_sig(t1->data[y][x], bpno + NMSEDEC_FRACBITS);
549                             ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, (t1->flags[y+1][x+1] >> 15) ^ xorbit);
550                             ff_jpeg2000_set_significance(t1, x, y, t1->flags[y+1][x+1] >> 15);
551                         }
552                     }
553                     t1->flags[y+1][x+1] &= ~JPEG2000_T1_VIS;
554                 }
555             } else{
556                 for (y = y0; y < y0 + 4 && y < height; y++){
557                     if (!(t1->flags[y+1][x+1] & (JPEG2000_T1_SIG | JPEG2000_T1_VIS))){
558                         int ctxno = ff_jpeg2000_getsigctxno(t1->flags[y+1][x+1], bandno);
559                         ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, t1->data[y][x] & mask ? 1:0);
560                         if (t1->data[y][x] & mask){ // newly significant
561                             int xorbit;
562                             int ctxno = ff_jpeg2000_getsgnctxno(t1->flags[y+1][x+1], &xorbit);
563                             *nmsedec += getnmsedec_sig(t1->data[y][x], bpno + NMSEDEC_FRACBITS);
564                             ff_mqc_encode(&t1->mqc, t1->mqc.cx_states + ctxno, (t1->flags[y+1][x+1] >> 15) ^ xorbit);
565                             ff_jpeg2000_set_significance(t1, x, y, t1->flags[y+1][x+1] >> 15);
566                         }
567                     }
568                     t1->flags[y+1][x+1] &= ~JPEG2000_T1_VIS;
569                 }
570             }
571         }
572 }
573
574 static void encode_cblk(Jpeg2000EncoderContext *s, Jpeg2000T1Context *t1, Jpeg2000Cblk *cblk, Jpeg2000Tile *tile,
575                         int width, int height, int bandpos, int lev)
576 {
577     int pass_t = 2, passno, x, y, max=0, nmsedec, bpno;
578     int64_t wmsedec = 0;
579
580     for (y = 0; y < height+2; y++)
581         memset(t1->flags[y], 0, (width+2)*sizeof(int));
582
583     for (y = 0; y < height; y++){
584         for (x = 0; x < width; x++){
585             if (t1->data[y][x] < 0){
586                 t1->flags[y+1][x+1] |= JPEG2000_T1_SGN;
587                 t1->data[y][x] = -t1->data[y][x];
588             }
589             max = FFMAX(max, t1->data[y][x]);
590         }
591     }
592
593     if (max == 0){
594         cblk->nonzerobits = 0;
595         bpno = 0;
596     } else{
597         cblk->nonzerobits = av_log2(max) + 1 - NMSEDEC_FRACBITS;
598         bpno = cblk->nonzerobits - 1;
599     }
600
601     ff_mqc_initenc(&t1->mqc, cblk->data);
602
603     for (passno = 0; bpno >= 0; passno++){
604         nmsedec=0;
605
606         switch(pass_t){
607             case 0: encode_sigpass(t1, width, height, bandpos, &nmsedec, bpno);
608                     break;
609             case 1: encode_refpass(t1, width, height, &nmsedec, bpno);
610                     break;
611             case 2: encode_clnpass(t1, width, height, bandpos, &nmsedec, bpno);
612                     break;
613         }
614
615         cblk->passes[passno].rate = ff_mqc_flush_to(&t1->mqc, cblk->passes[passno].flushed, &cblk->passes[passno].flushed_len);
616         wmsedec += (int64_t)nmsedec << (2*bpno);
617         cblk->passes[passno].disto = wmsedec;
618
619         if (++pass_t == 3){
620             pass_t = 0;
621             bpno--;
622         }
623     }
624     cblk->npasses = passno;
625     cblk->ninclpasses = passno;
626
627     cblk->passes[passno-1].rate = ff_mqc_flush_to(&t1->mqc, cblk->passes[passno-1].flushed, &cblk->passes[passno-1].flushed_len);
628 }
629
630 /* tier-2 routines: */
631
632 static void putnumpasses(Jpeg2000EncoderContext *s, int n)
633 {
634     if (n == 1)
635         put_num(s, 0, 1);
636     else if (n == 2)
637         put_num(s, 2, 2);
638     else if (n <= 5)
639         put_num(s, 0xc | (n-3), 4);
640     else if (n <= 36)
641         put_num(s, 0x1e0 | (n-6), 9);
642     else
643         put_num(s, 0xff80 | (n-37), 16);
644 }
645
646
647 static int encode_packet(Jpeg2000EncoderContext *s, Jpeg2000ResLevel *rlevel, int precno,
648                           uint8_t *expn, int numgbits)
649 {
650     int bandno, empty = 1;
651
652     // init bitstream
653     *s->buf = 0;
654     s->bit_index = 0;
655
656     // header
657
658     // is the packet empty?
659     for (bandno = 0; bandno < rlevel->nbands; bandno++){
660         if (rlevel->band[bandno].coord[0][0] < rlevel->band[bandno].coord[0][1]
661         &&  rlevel->band[bandno].coord[1][0] < rlevel->band[bandno].coord[1][1]){
662             empty = 0;
663             break;
664         }
665     }
666
667     put_bits(s, !empty, 1);
668     if (empty){
669         j2k_flush(s);
670         return 0;
671     }
672
673     for (bandno = 0; bandno < rlevel->nbands; bandno++){
674         Jpeg2000Band *band = rlevel->band + bandno;
675         Jpeg2000Prec *prec = band->prec + precno;
676         int yi, xi, pos;
677         int cblknw = prec->nb_codeblocks_width;
678
679         if (band->coord[0][0] == band->coord[0][1]
680         ||  band->coord[1][0] == band->coord[1][1])
681             continue;
682
683         for (pos=0, yi = 0; yi < prec->nb_codeblocks_height; yi++){
684             for (xi = 0; xi < cblknw; xi++, pos++){
685                 prec->cblkincl[pos].val = prec->cblk[yi * cblknw + xi].ninclpasses == 0;
686                 tag_tree_update(prec->cblkincl + pos);
687                 prec->zerobits[pos].val = expn[bandno] + numgbits - 1 - prec->cblk[yi * cblknw + xi].nonzerobits;
688                 tag_tree_update(prec->zerobits + pos);
689             }
690         }
691
692         for (pos=0, yi = 0; yi < prec->nb_codeblocks_height; yi++){
693             for (xi = 0; xi < cblknw; xi++, pos++){
694                 int pad = 0, llen, length;
695                 Jpeg2000Cblk *cblk = prec->cblk + yi * cblknw + xi;
696
697                 if (s->buf_end - s->buf < 20) // approximately
698                     return -1;
699
700                 // inclusion information
701                 tag_tree_code(s, prec->cblkincl + pos, 1);
702                 if (!cblk->ninclpasses)
703                     continue;
704                 // zerobits information
705                 tag_tree_code(s, prec->zerobits + pos, 100);
706                 // number of passes
707                 putnumpasses(s, cblk->ninclpasses);
708
709                 length = cblk->passes[cblk->ninclpasses-1].rate;
710                 llen = av_log2(length) - av_log2(cblk->ninclpasses) - 2;
711                 if (llen < 0){
712                     pad = -llen;
713                     llen = 0;
714                 }
715                 // length of code block
716                 put_bits(s, 1, llen);
717                 put_bits(s, 0, 1);
718                 put_num(s, length, av_log2(length)+1+pad);
719             }
720         }
721     }
722     j2k_flush(s);
723     for (bandno = 0; bandno < rlevel->nbands; bandno++){
724         Jpeg2000Band *band = rlevel->band + bandno;
725         Jpeg2000Prec *prec = band->prec + precno;
726         int yi, cblknw = prec->nb_codeblocks_width;
727         for (yi =0; yi < prec->nb_codeblocks_height; yi++){
728             int xi;
729             for (xi = 0; xi < cblknw; xi++){
730                 Jpeg2000Cblk *cblk = prec->cblk + yi * cblknw + xi;
731                 if (cblk->ninclpasses){
732                     if (s->buf_end - s->buf < cblk->passes[cblk->ninclpasses-1].rate)
733                         return -1;
734                     bytestream_put_buffer(&s->buf, cblk->data,   cblk->passes[cblk->ninclpasses-1].rate
735                                                                - cblk->passes[cblk->ninclpasses-1].flushed_len);
736                     bytestream_put_buffer(&s->buf, cblk->passes[cblk->ninclpasses-1].flushed,
737                                                    cblk->passes[cblk->ninclpasses-1].flushed_len);
738                 }
739             }
740         }
741     }
742     return 0;
743 }
744
745 static int encode_packets(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno)
746 {
747     int compno, reslevelno, ret;
748     Jpeg2000CodingStyle *codsty = &s->codsty;
749     Jpeg2000QuantStyle  *qntsty = &s->qntsty;
750
751     av_log(s->avctx, AV_LOG_DEBUG, "tier2\n");
752     // lay-rlevel-comp-pos progression
753     for (reslevelno = 0; reslevelno < codsty->nreslevels; reslevelno++){
754         for (compno = 0; compno < s->ncomponents; compno++){
755             int precno;
756             Jpeg2000ResLevel *reslevel = s->tile[tileno].comp[compno].reslevel + reslevelno;
757             for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++){
758                 if ((ret = encode_packet(s, reslevel, precno, qntsty->expn + (reslevelno ? 3*reslevelno-2 : 0),
759                               qntsty->nguardbits)) < 0)
760                     return ret;
761             }
762         }
763     }
764     av_log(s->avctx, AV_LOG_DEBUG, "after tier2\n");
765     return 0;
766 }
767
768 static int getcut(Jpeg2000Cblk *cblk, int64_t lambda, int dwt_norm)
769 {
770     int passno, res = 0;
771     for (passno = 0; passno < cblk->npasses; passno++){
772         int dr;
773         int64_t dd;
774
775         dr = cblk->passes[passno].rate
776            - (res ? cblk->passes[res-1].rate:0);
777         dd = cblk->passes[passno].disto
778            - (res ? cblk->passes[res-1].disto:0);
779
780         if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda)
781             res = passno+1;
782     }
783     return res;
784 }
785
786 static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile)
787 {
788     int precno, compno, reslevelno, bandno, cblkno, lev;
789     Jpeg2000CodingStyle *codsty = &s->codsty;
790
791     for (compno = 0; compno < s->ncomponents; compno++){
792         Jpeg2000Component *comp = tile->comp + compno;
793
794         for (reslevelno = 0, lev = codsty->nreslevels-1; reslevelno < codsty->nreslevels; reslevelno++, lev--){
795             Jpeg2000ResLevel *reslevel = comp->reslevel + reslevelno;
796
797             for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++){
798                 for (bandno = 0; bandno < reslevel->nbands ; bandno++){
799                     int bandpos = bandno + (reslevelno > 0);
800                     Jpeg2000Band *band = reslevel->band + bandno;
801                     Jpeg2000Prec *prec = band->prec + precno;
802
803                     for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){
804                         Jpeg2000Cblk *cblk = prec->cblk + cblkno;
805
806                         cblk->ninclpasses = getcut(cblk, s->lambda,
807                                 (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15);
808                     }
809                 }
810             }
811         }
812     }
813 }
814
815 static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno)
816 {
817     int compno, reslevelno, bandno, ret;
818     Jpeg2000T1Context t1;
819     Jpeg2000CodingStyle *codsty = &s->codsty;
820     for (compno = 0; compno < s->ncomponents; compno++){
821         Jpeg2000Component *comp = s->tile[tileno].comp + compno;
822
823         av_log(s->avctx, AV_LOG_DEBUG,"dwt\n");
824         if ((ret = ff_dwt_encode(&comp->dwt, comp->i_data)) < 0)
825             return ret;
826         av_log(s->avctx, AV_LOG_DEBUG,"after dwt -> tier1\n");
827
828         for (reslevelno = 0; reslevelno < codsty->nreslevels; reslevelno++){
829             Jpeg2000ResLevel *reslevel = comp->reslevel + reslevelno;
830
831             for (bandno = 0; bandno < reslevel->nbands ; bandno++){
832                 Jpeg2000Band *band = reslevel->band + bandno;
833                 Jpeg2000Prec *prec = band->prec; // we support only 1 precinct per band ATM in the encoder
834                 int cblkx, cblky, cblkno=0, xx0, x0, xx1, y0, yy0, yy1, bandpos;
835                 yy0 = bandno == 0 ? 0 : comp->reslevel[reslevelno-1].coord[1][1] - comp->reslevel[reslevelno-1].coord[1][0];
836                 y0 = yy0;
837                 yy1 = FFMIN(ff_jpeg2000_ceildivpow2(band->coord[1][0] + 1, band->log2_cblk_height) << band->log2_cblk_height,
838                             band->coord[1][1]) - band->coord[1][0] + yy0;
839
840                 if (band->coord[0][0] == band->coord[0][1] || band->coord[1][0] == band->coord[1][1])
841                     continue;
842
843                 bandpos = bandno + (reslevelno > 0);
844
845                 for (cblky = 0; cblky < prec->nb_codeblocks_height; cblky++){
846                     if (reslevelno == 0 || bandno == 1)
847                         xx0 = 0;
848                     else
849                         xx0 = comp->reslevel[reslevelno-1].coord[0][1] - comp->reslevel[reslevelno-1].coord[0][0];
850                     x0 = xx0;
851                     xx1 = FFMIN(ff_jpeg2000_ceildivpow2(band->coord[0][0] + 1, band->log2_cblk_width) << band->log2_cblk_width,
852                                 band->coord[0][1]) - band->coord[0][0] + xx0;
853
854                     for (cblkx = 0; cblkx < prec->nb_codeblocks_width; cblkx++, cblkno++){
855                         int y, x;
856                         if (codsty->transform == FF_DWT53){
857                             for (y = yy0; y < yy1; y++){
858                                 int *ptr = t1.data[y-yy0];
859                                 for (x = xx0; x < xx1; x++){
860                                     *ptr++ = comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] << NMSEDEC_FRACBITS;
861                                 }
862                             }
863                         } else{
864                             for (y = yy0; y < yy1; y++){
865                                 int *ptr = t1.data[y-yy0];
866                                 for (x = xx0; x < xx1; x++){
867                                     *ptr = (comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x]);
868                                     *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 15 - NMSEDEC_FRACBITS;
869                                     ptr++;
870                                 }
871                             }
872                         }
873                         encode_cblk(s, &t1, prec->cblk + cblkno, tile, xx1 - xx0, yy1 - yy0,
874                                     bandpos, codsty->nreslevels - reslevelno - 1);
875                         xx0 = xx1;
876                         xx1 = FFMIN(xx1 + (1 << band->log2_cblk_width), band->coord[0][1] - band->coord[0][0] + x0);
877                     }
878                     yy0 = yy1;
879                     yy1 = FFMIN(yy1 + (1 << band->log2_cblk_height), band->coord[1][1] - band->coord[1][0] + y0);
880                 }
881             }
882         }
883         av_log(s->avctx, AV_LOG_DEBUG, "after tier1\n");
884     }
885
886     av_log(s->avctx, AV_LOG_DEBUG, "rate control\n");
887     truncpasses(s, tile);
888     if ((ret = encode_packets(s, tile, tileno)) < 0)
889         return ret;
890     av_log(s->avctx, AV_LOG_DEBUG, "after rate control\n");
891     return 0;
892 }
893
894 static void cleanup(Jpeg2000EncoderContext *s)
895 {
896     int tileno, compno;
897     Jpeg2000CodingStyle *codsty = &s->codsty;
898
899     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
900         for (compno = 0; compno < s->ncomponents; compno++){
901             Jpeg2000Component *comp = s->tile[tileno].comp + compno;
902             ff_jpeg2000_cleanup(comp, codsty);
903         }
904         av_freep(&s->tile[tileno].comp);
905     }
906     av_freep(&s->tile);
907 }
908
909 static void reinit(Jpeg2000EncoderContext *s)
910 {
911     int tileno, compno;
912     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
913         Jpeg2000Tile *tile = s->tile + tileno;
914         for (compno = 0; compno < s->ncomponents; compno++)
915             ff_jpeg2000_reinit(tile->comp + compno, &s->codsty);
916     }
917 }
918
919 static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
920                         const AVFrame *pict, int *got_packet)
921 {
922     int tileno, ret;
923     Jpeg2000EncoderContext *s = avctx->priv_data;
924
925     if ((ret = ff_alloc_packet2(avctx, pkt, avctx->width*avctx->height*9 + FF_MIN_BUFFER_SIZE)) < 0)
926         return ret;
927
928     // init:
929     s->buf = s->buf_start = pkt->data;
930     s->buf_end = pkt->data + pkt->size;
931
932     s->picture = pict;
933
934     s->lambda = s->picture->quality * LAMBDA_SCALE;
935
936     copy_frame(s);
937     reinit(s);
938
939     if (s->buf_end - s->buf < 2)
940         return -1;
941     bytestream_put_be16(&s->buf, JPEG2000_SOC);
942     if ((ret = put_siz(s)) < 0)
943         return ret;
944     if ((ret = put_cod(s)) < 0)
945         return ret;
946     if ((ret = put_qcd(s, 0)) < 0)
947         return ret;
948
949     for (tileno = 0; tileno < s->numXtiles * s->numYtiles; tileno++){
950         uint8_t *psotptr;
951         if (!(psotptr = put_sot(s, tileno)))
952             return -1;
953         if (s->buf_end - s->buf < 2)
954             return -1;
955         bytestream_put_be16(&s->buf, JPEG2000_SOD);
956         if ((ret = encode_tile(s, s->tile + tileno, tileno)) < 0)
957             return ret;
958         bytestream_put_be32(&psotptr, s->buf - psotptr + 6);
959     }
960     if (s->buf_end - s->buf < 2)
961         return -1;
962     bytestream_put_be16(&s->buf, JPEG2000_EOC);
963
964     av_log(s->avctx, AV_LOG_DEBUG, "end\n");
965     pkt->size = s->buf - s->buf_start;
966     pkt->flags |= AV_PKT_FLAG_KEY;
967     *got_packet = 1;
968
969     return 0;
970 }
971
972 static av_cold int j2kenc_init(AVCodecContext *avctx)
973 {
974     int i, ret;
975     Jpeg2000EncoderContext *s = avctx->priv_data;
976     Jpeg2000CodingStyle *codsty = &s->codsty;
977     Jpeg2000QuantStyle  *qntsty = &s->qntsty;
978
979     s->avctx = avctx;
980     av_log(s->avctx, AV_LOG_DEBUG, "init\n");
981
982     // defaults:
983     // TODO: implement setting non-standard precinct size
984     memset(codsty->log2_prec_widths , 15, sizeof(codsty->log2_prec_widths ));
985     memset(codsty->log2_prec_heights, 15, sizeof(codsty->log2_prec_heights));
986     codsty->nreslevels2decode=
987     codsty->nreslevels       = 7;
988     codsty->log2_cblk_width  = 4;
989     codsty->log2_cblk_height = 4;
990     codsty->transform        = avctx->prediction_method ? FF_DWT53 : FF_DWT97_INT;
991
992     qntsty->nguardbits       = 1;
993
994     s->tile_width            = 256;
995     s->tile_height           = 256;
996
997     if (codsty->transform == FF_DWT53)
998         qntsty->quantsty = JPEG2000_QSTY_NONE;
999     else
1000         qntsty->quantsty = JPEG2000_QSTY_SE;
1001
1002     s->width = avctx->width;
1003     s->height = avctx->height;
1004
1005     for (i = 0; i < 3; i++)
1006         s->cbps[i] = 8;
1007
1008     if (avctx->pix_fmt == AV_PIX_FMT_RGB24){
1009         s->ncomponents = 3;
1010     } else if (avctx->pix_fmt == AV_PIX_FMT_GRAY8){
1011         s->ncomponents = 1;
1012     } else{ // planar YUV
1013         s->planar = 1;
1014         s->ncomponents = 3;
1015         avcodec_get_chroma_sub_sample(avctx->pix_fmt,
1016                 s->chroma_shift, s->chroma_shift + 1);
1017     }
1018
1019     ff_jpeg2000_init_tier1_luts();
1020     ff_mqc_init_context_tables();
1021     init_luts();
1022
1023     init_quantization(s);
1024     if ((ret=init_tiles(s)) < 0)
1025         return ret;
1026
1027     av_log(s->avctx, AV_LOG_DEBUG, "after init\n");
1028
1029     return 0;
1030 }
1031
1032 static int j2kenc_destroy(AVCodecContext *avctx)
1033 {
1034     Jpeg2000EncoderContext *s = avctx->priv_data;
1035
1036     cleanup(s);
1037     return 0;
1038 }
1039
1040 AVCodec ff_jpeg2000_encoder = {
1041     .name           = "jpeg2000",
1042     .long_name      = NULL_IF_CONFIG_SMALL("JPEG 2000"),
1043     .type           = AVMEDIA_TYPE_VIDEO,
1044     .id             = AV_CODEC_ID_JPEG2000,
1045     .priv_data_size = sizeof(Jpeg2000EncoderContext),
1046     .init           = j2kenc_init,
1047     .encode2        = encode_frame,
1048     .close          = j2kenc_destroy,
1049     .capabilities   = CODEC_CAP_EXPERIMENTAL,
1050     .pix_fmts       = (const enum AVPixelFormat[]) {
1051         AV_PIX_FMT_RGB24, AV_PIX_FMT_YUV444P, AV_PIX_FMT_GRAY8,
1052 /*      AV_PIX_FMT_YUV420P,
1053         AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P,
1054         AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P,*/
1055         AV_PIX_FMT_NONE
1056     }
1057 };