avcodec: va: directly set data[3] with surface
authorAlexandre Janniaux <ajanni@videolabs.io>
Tue, 23 Jun 2020 16:51:34 +0000 (18:51 +0200)
committerAlexandre Janniaux <ajanni@videolabs.io>
Sat, 27 Jun 2020 10:12:46 +0000 (12:12 +0200)
We never use data[0] for anything, nor does avcodec, and it can
actually keep the same initialized value or be NULL.

In particular, it fixes avcodec decoder with vaapi vlc_va_t backend
which was generating VASurfaceID of value 0 to count, leading to the
following assertion:

    vlc: ../../modules/codec/avcodec/video.c:1492: lavc_va_GetFrame: Assertion `frame->data[0] != NULL' failed.
    [1]    154639 abort (core dumped)  ./build-native/vlc <sample> --dec-dev=vaapi -vvv

modules/codec/avcodec/d3d11va.c
modules/codec/avcodec/dxva2.c
modules/codec/avcodec/vaapi.c
modules/codec/avcodec/video.c

index aaa41fc..c9793a5 100644 (file)
@@ -227,7 +227,7 @@ static int Get(vlc_va_t *va, picture_t *pic, uint8_t **data)
         va_surface_Release(va_surface);
         return VLC_ENOITEM;
     }
-    *data = (uint8_t*)sys->hw_surface[va_surface_GetIndex(va_surface)];
+    data[3] = (uint8_t*)sys->hw_surface[va_surface_GetIndex(va_surface)];
     return VLC_SUCCESS;
 }
 
index aefcb55..4a7b628 100644 (file)
@@ -238,7 +238,7 @@ static int Get(vlc_va_t *va, picture_t *pic, uint8_t **data)
         va_surface_Release(va_surface);
         return VLC_ENOITEM;
     }
-    *data = (uint8_t*)DXVA2_PICCONTEXT_FROM_PICCTX(pic->context)->ctx.picsys.surface;
+    data[3] = (uint8_t*)DXVA2_PICCONTEXT_FROM_PICCTX(pic->context)->ctx.picsys.surface;
     return VLC_SUCCESS;
 }
 
index 41136dc..01d2b35 100644 (file)
@@ -170,7 +170,7 @@ static int Get(vlc_va_t *va, picture_t *pic, uint8_t **data)
     vaapi_ctx->ctx.va_dpy = sys->hw_ctx.display;
     vaapi_ctx->va_surface = va_surface;
     vlc_vaapi_PicSetContext(pic, &vaapi_ctx->ctx);
-    *data = (void *) (uintptr_t) vaapi_ctx->ctx.surface;
+    data[3] = (void *) (uintptr_t) vaapi_ctx->ctx.surface;
 
     return VLC_SUCCESS;
 }
index b74d77e..d89635d 100644 (file)
@@ -1483,16 +1483,13 @@ static int lavc_va_GetFrame(struct AVCodecContext *ctx, AVFrame *frame)
     if (pic == NULL)
         return -1;
 
+    /* data[3] will contains the format-specific surface handle. */
     if (vlc_va_Get(va, pic, &frame->data[0]))
     {
         msg_Err(dec, "hardware acceleration picture allocation failed");
         picture_Release(pic);
         return -1;
     }
-    assert(frame->data[0] != NULL);
-    /* data[0] must be non-NULL for libavcodec internal checks.
-     * data[3] actually contains the format-specific surface handle. */
-    frame->data[3] = frame->data[0];
 
     frame->buf[0] = av_buffer_create(frame->data[0], 0, lavc_ReleaseFrame, pic, 0);
     if (unlikely(frame->buf[0] == NULL))