lavfi: detect merge failure for unknown layouts.
authorNicolas George <nicolas.george@normalesup.org>
Thu, 14 Mar 2013 20:54:48 +0000 (21:54 +0100)
committerNicolas George <nicolas.george@normalesup.org>
Mon, 1 Apr 2013 08:37:48 +0000 (10:37 +0200)
Detect when filtering known layouts from an explicit list
results in an empty list.

Fix erratic behavior.

libavfilter/formats.c

index 43718e4..ea24627 100644 (file)
@@ -184,6 +184,10 @@ AVFilterChannelLayouts *ff_merge_channel_layouts(AVFilterChannelLayouts *a,
             for (i = j = 0; i < b->nb_channel_layouts; i++)
                 if (KNOWN(b->channel_layouts[i]))
                     b->channel_layouts[j++] = b->channel_layouts[i];
+            /* Not optimal: the unknown layouts of b may become known after
+               another merge. */
+            if (!j)
+                return NULL;
             b->nb_channel_layouts = j;
         }
         MERGE_REF(b, a, channel_layouts, AVFilterChannelLayouts, fail);