lavfi/mergeplanes: Fix >8 bit for big endian formats and yuv4xxp16le.
authorCarl Eugen Hoyos <cehoyos@ag.or.at>
Sat, 29 Oct 2016 22:25:12 +0000 (00:25 +0200)
committerCarl Eugen Hoyos <cehoyos@ag.or.at>
Tue, 1 Nov 2016 11:24:57 +0000 (12:24 +0100)
Fixes part of ticket #5916.

libavfilter/vf_mergeplanes.c

index 8128f33..c4948cc 100644 (file)
@@ -192,9 +192,9 @@ static int config_output(AVFilterLink *outlink)
     outlink->sample_aspect_ratio = ctx->inputs[0]->sample_aspect_ratio;
 
     s->planewidth[1]  =
-    s->planewidth[2]  = AV_CEIL_RSHIFT(outlink->w, s->outdesc->log2_chroma_w);
+    s->planewidth[2]  = AV_CEIL_RSHIFT(((s->outdesc->comp[1].depth > 8) + 1) * outlink->w, s->outdesc->log2_chroma_w);
     s->planewidth[0]  =
-    s->planewidth[3]  = outlink->w;
+    s->planewidth[3]  = ((s->outdesc->comp[0].depth > 8) + 1) * outlink->w;
     s->planeheight[1] =
     s->planeheight[2] = AV_CEIL_RSHIFT(outlink->h, s->outdesc->log2_chroma_h);
     s->planeheight[0] =
@@ -220,9 +220,9 @@ static int config_output(AVFilterLink *outlink)
         }
 
         inputp->planewidth[1]  =
-        inputp->planewidth[2]  = AV_CEIL_RSHIFT(inlink->w, indesc->log2_chroma_w);
+        inputp->planewidth[2]  = AV_CEIL_RSHIFT(((indesc->comp[1].depth > 8) + 1) * inlink->w, indesc->log2_chroma_w);
         inputp->planewidth[0]  =
-        inputp->planewidth[3]  = inlink->w;
+        inputp->planewidth[3]  = ((indesc->comp[0].depth > 8) + 1) * inlink->w;
         inputp->planeheight[1] =
         inputp->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, indesc->log2_chroma_h);
         inputp->planeheight[0] =