d3d11va: don't use the external pool for NVIDIA hardware with too many slices
authorSteve Lhomme <robux4@videolabs.io>
Mon, 29 May 2017 10:48:17 +0000 (12:48 +0200)
committerJean-Baptiste Kempf <jb@videolan.org>
Tue, 13 Jun 2017 12:44:03 +0000 (14:44 +0200)
Drivers crash during ID3D11VideoDevice::CreateVideoDecoderOutputView() if a
texture has more than 30 slices.

Fixes #18261

Signed-off-by: Jean-Baptiste Kempf <jb@videolan.org>
modules/codec/avcodec/d3d11va.c
modules/video_chroma/dxgi_fmt.c
modules/video_chroma/dxgi_fmt.h

index 034beb0..46f95c1 100644 (file)
@@ -110,6 +110,7 @@ struct vlc_va_sys_t
 {
     directx_sys_t                dx_sys;
     vlc_fourcc_t                 i_chroma;
+    UINT                         totalTextureSlices;
 
 #if !defined(NDEBUG) && defined(HAVE_DXGIDEBUG_H)
     HINSTANCE                    dxgidebug_dll;
@@ -434,12 +435,12 @@ static int Open(vlc_va_t *va, AVCodecContext *ctx, enum PixelFormat pix_fmt,
 
             sys->d3dctx = p_sys->context;
             sys->d3dvidctx = d3dvidctx;
-            sys->b_extern_pool = true;
 
             assert(p_sys->texture[KNOWN_DXGI_INDEX] != NULL);
             D3D11_TEXTURE2D_DESC dstDesc;
             ID3D11Texture2D_GetDesc( p_sys->texture[KNOWN_DXGI_INDEX], &dstDesc);
             sys->render = dstDesc.Format;
+            va->sys->totalTextureSlices = dstDesc.ArraySize;
         }
     }
 
@@ -825,6 +826,16 @@ static int DxSetupOutput(vlc_va_t *va, const GUID *input, const video_format_t *
         }
 
         msg_Dbg(va, "Using output format %s for decoder %s", DxgiFormatToStr(processorInput[idx]), psz_decoder_name);
+        if ( va->sys->render == processorInput[idx] )
+        {
+            /* NVIDIA cards crash when calling CreateVideoDecoderOutputView
+             * on more than 30 slices */
+            if (va->sys->totalTextureSlices <= 30 || !isNvidiaHardware(dx_sys->d3ddev))
+                va->sys->b_extern_pool = true;
+            else
+                msg_Warn( va, "NVIDIA GPU with too many slices (%d) detected, use internal pool",
+                          va->sys->totalTextureSlices );
+        }
         va->sys->render = processorInput[idx];
         free(psz_decoder_name);
         return VLC_SUCCESS;
index 9fbf702..f496e98 100644 (file)
@@ -160,3 +160,19 @@ bool isXboxHardware(ID3D11Device *d3ddev)
     IDXGIAdapter_Release(p_adapter);
     return result;
 }
+
+bool isNvidiaHardware(ID3D11Device *d3ddev)
+{
+    IDXGIAdapter *p_adapter = D3D11DeviceAdapter(d3ddev);
+    if (!p_adapter)
+        return NULL;
+
+    bool result = false;
+    DXGI_ADAPTER_DESC adapterDesc;
+    if (SUCCEEDED(IDXGIAdapter_GetDesc(p_adapter, &adapterDesc))) {
+        result = adapterDesc.VendorId == 0x10DE;
+    }
+
+    IDXGIAdapter_Release(p_adapter);
+    return result;
+}
index f20e488..2e4d987 100644 (file)
@@ -49,6 +49,7 @@ extern void DxgiFormatMask(DXGI_FORMAT format, video_format_t *);
 
 typedef struct ID3D11Device ID3D11Device;
 bool isXboxHardware(ID3D11Device *d3ddev);
+bool isNvidiaHardware(ID3D11Device *d3ddev);
 IDXGIAdapter *D3D11DeviceAdapter(ID3D11Device *d3ddev);
 
 static inline bool DeviceSupportsFormat(ID3D11Device *d3ddevice,