Merge commit 'e18ba2dfd2d19aedc8afccf011d5fd0833352423'
authorHendrik Leppkes <h.leppkes@gmail.com>
Sat, 1 Apr 2017 16:53:40 +0000 (18:53 +0200)
committerHendrik Leppkes <h.leppkes@gmail.com>
Sat, 1 Apr 2017 16:53:40 +0000 (18:53 +0200)
* commit 'e18ba2dfd2d19aedc8afccf011d5fd0833352423':
  hwcontext_dxva2: make sure the sw frame format is the right one during transfer

Merged-by: Hendrik Leppkes <h.leppkes@gmail.com>
1  2 
libavutil/hwcontext_dxva2.c

@@@ -1,18 -1,18 +1,18 @@@
  /*
 - * This file is part of Libav.
 + * This file is part of FFmpeg.
   *
 - * Libav is free software; you can redistribute it and/or
 + * FFmpeg is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
   * version 2.1 of the License, or (at your option) any later version.
   *
 - * Libav is distributed in the hope that it will be useful,
 + * FFmpeg is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   * Lesser General Public License for more details.
   *
   * You should have received a copy of the GNU Lesser General Public
 - * License along with Libav; if not, write to the Free Software
 + * License along with FFmpeg; if not, write to the Free Software
   * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
   */
  
@@@ -29,7 -29,6 +29,7 @@@
  #include <dxva2api.h>
  #include <initguid.h>
  
 +#include "avassert.h"
  #include "common.h"
  #include "hwcontext.h"
  #include "hwcontext_dxva2.h"
  #include "imgutils.h"
  #include "pixdesc.h"
  #include "pixfmt.h"
 +#include "compat/w32dlfcn.h"
  
  typedef IDirect3D9* WINAPI pDirect3DCreate9(UINT);
 +typedef HRESULT WINAPI pDirect3DCreate9Ex(UINT, IDirect3D9Ex **);
  typedef HRESULT WINAPI pCreateDeviceManager9(UINT *, IDirect3DDeviceManager9 **);
  
 +#define FF_D3DCREATE_FLAGS (D3DCREATE_SOFTWARE_VERTEXPROCESSING | \
 +                            D3DCREATE_MULTITHREADED | \
 +                            D3DCREATE_FPU_PRESERVE)
 +
 +static const D3DPRESENT_PARAMETERS dxva2_present_params = {
 +    .Windowed         = TRUE,
 +    .BackBufferWidth  = 640,
 +    .BackBufferHeight = 480,
 +    .BackBufferCount  = 0,
 +    .SwapEffect       = D3DSWAPEFFECT_DISCARD,
 +    .Flags            = D3DPRESENTFLAG_VIDEO,
 +};
 +
  typedef struct DXVA2Mapping {
      uint32_t palette_dummy[256];
  } DXVA2Mapping;
@@@ -331,6 -315,9 +331,9 @@@ static int dxva2_transfer_data_to(AVHWF
      AVFrame *map;
      int ret;
  
+     if (src->format != ctx->sw_format)
+         return AVERROR(ENOSYS);
      map = av_frame_alloc();
      if (!map)
          return AVERROR(ENOMEM);
@@@ -355,6 -342,9 +358,9 @@@ static int dxva2_transfer_data_from(AVH
      ptrdiff_t src_linesize[4], dst_linesize[4];
      int ret, i;
  
+     if (dst->format != ctx->sw_format)
+         return AVERROR(ENOSYS);
      map = av_frame_alloc();
      if (!map)
          return AVERROR(ENOMEM);
@@@ -413,91 -403,27 +419,91 @@@ static void dxva2_device_free(AVHWDevic
          IDirect3D9_Release(priv->d3d9);
  
      if (priv->d3dlib)
 -        FreeLibrary(priv->d3dlib);
 +        dlclose(priv->d3dlib);
  
      if (priv->dxva2lib)
 -        FreeLibrary(priv->dxva2lib);
 +        dlclose(priv->dxva2lib);
  
      av_freep(&ctx->user_opaque);
  }
  
 +static int dxva2_device_create9(AVHWDeviceContext *ctx, UINT adapter)
 +{
 +    DXVA2DevicePriv *priv = ctx->user_opaque;
 +    D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
 +    D3DDISPLAYMODE d3ddm;
 +    HRESULT hr;
 +    pDirect3DCreate9 *createD3D = (pDirect3DCreate9 *)dlsym(priv->d3dlib, "Direct3DCreate9");
 +    if (!createD3D) {
 +        av_log(ctx, AV_LOG_ERROR, "Failed to locate Direct3DCreate9\n");
 +        return AVERROR_UNKNOWN;
 +    }
 +
 +    priv->d3d9 = createD3D(D3D_SDK_VERSION);
 +    if (!priv->d3d9) {
 +        av_log(ctx, AV_LOG_ERROR, "Failed to create IDirect3D object\n");
 +        return AVERROR_UNKNOWN;
 +    }
 +
 +    IDirect3D9_GetAdapterDisplayMode(priv->d3d9, adapter, &d3ddm);
 +
 +    d3dpp.BackBufferFormat = d3ddm.Format;
 +
 +    hr = IDirect3D9_CreateDevice(priv->d3d9, adapter, D3DDEVTYPE_HAL, GetDesktopWindow(),
 +                                 FF_D3DCREATE_FLAGS,
 +                                 &d3dpp, &priv->d3d9device);
 +    if (FAILED(hr)) {
 +        av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device\n");
 +        return AVERROR_UNKNOWN;
 +    }
 +
 +    return 0;
 +}
 +
 +static int dxva2_device_create9ex(AVHWDeviceContext *ctx, UINT adapter)
 +{
 +    DXVA2DevicePriv *priv = ctx->user_opaque;
 +    D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
 +    D3DDISPLAYMODEEX modeex = {0};
 +    IDirect3D9Ex *d3d9ex = NULL;
 +    IDirect3DDevice9Ex *exdev = NULL;
 +    HRESULT hr;
 +    pDirect3DCreate9Ex *createD3DEx = (pDirect3DCreate9Ex *)dlsym(priv->d3dlib, "Direct3DCreate9Ex");
 +    if (!createD3DEx)
 +        return AVERROR(ENOSYS);
 +
 +    hr = createD3DEx(D3D_SDK_VERSION, &d3d9ex);
 +    if (FAILED(hr))
 +        return AVERROR_UNKNOWN;
 +
 +    IDirect3D9Ex_GetAdapterDisplayModeEx(d3d9ex, adapter, &modeex, NULL);
 +
 +    d3dpp.BackBufferFormat = modeex.Format;
 +
 +    hr = IDirect3D9Ex_CreateDeviceEx(d3d9ex, adapter, D3DDEVTYPE_HAL, GetDesktopWindow(),
 +                                     FF_D3DCREATE_FLAGS,
 +                                     &d3dpp, NULL, &exdev);
 +    if (FAILED(hr)) {
 +        IDirect3D9Ex_Release(d3d9ex);
 +        return AVERROR_UNKNOWN;
 +    }
 +
 +    av_log(ctx, AV_LOG_VERBOSE, "Using D3D9Ex device.\n");
 +    priv->d3d9 = (IDirect3D9 *)d3d9ex;
 +    priv->d3d9device = (IDirect3DDevice9 *)exdev;
 +    return 0;
 +}
 +
  static int dxva2_device_create(AVHWDeviceContext *ctx, const char *device,
                                 AVDictionary *opts, int flags)
  {
      AVDXVA2DeviceContext *hwctx = ctx->hwctx;
      DXVA2DevicePriv *priv;
 -
 -    pDirect3DCreate9 *createD3D = NULL;
      pCreateDeviceManager9 *createDeviceManager = NULL;
 -    D3DPRESENT_PARAMETERS d3dpp = {0};
 -    D3DDISPLAYMODE        d3ddm;
      unsigned resetToken = 0;
      UINT adapter = D3DADAPTER_DEFAULT;
      HRESULT hr;
 +    int err;
  
      if (device)
          adapter = atoi(device);
  
      priv->device_handle = INVALID_HANDLE_VALUE;
  
 -    priv->d3dlib = LoadLibrary("d3d9.dll");
 +    priv->d3dlib = dlopen("d3d9.dll", 0);
      if (!priv->d3dlib) {
          av_log(ctx, AV_LOG_ERROR, "Failed to load D3D9 library\n");
          return AVERROR_UNKNOWN;
      }
 -    priv->dxva2lib = LoadLibrary("dxva2.dll");
 +    priv->dxva2lib = dlopen("dxva2.dll", 0);
      if (!priv->dxva2lib) {
          av_log(ctx, AV_LOG_ERROR, "Failed to load DXVA2 library\n");
          return AVERROR_UNKNOWN;
      }
  
 -    createD3D = (pDirect3DCreate9 *)GetProcAddress(priv->d3dlib, "Direct3DCreate9");
 -    if (!createD3D) {
 -        av_log(ctx, AV_LOG_ERROR, "Failed to locate Direct3DCreate9\n");
 -        return AVERROR_UNKNOWN;
 -    }
 -    createDeviceManager = (pCreateDeviceManager9 *)GetProcAddress(priv->dxva2lib,
 -                                                                  "DXVA2CreateDirect3DDeviceManager9");
 +    createDeviceManager = (pCreateDeviceManager9 *)dlsym(priv->dxva2lib,
 +                                                         "DXVA2CreateDirect3DDeviceManager9");
      if (!createDeviceManager) {
          av_log(ctx, AV_LOG_ERROR, "Failed to locate DXVA2CreateDirect3DDeviceManager9\n");
          return AVERROR_UNKNOWN;
      }
  
 -    priv->d3d9 = createD3D(D3D_SDK_VERSION);
 -    if (!priv->d3d9) {
 -        av_log(ctx, AV_LOG_ERROR, "Failed to create IDirect3D object\n");
 -        return AVERROR_UNKNOWN;
 -    }
 -
 -    IDirect3D9_GetAdapterDisplayMode(priv->d3d9, adapter, &d3ddm);
 -    d3dpp.Windowed         = TRUE;
 -    d3dpp.BackBufferWidth  = 640;
 -    d3dpp.BackBufferHeight = 480;
 -    d3dpp.BackBufferCount  = 0;
 -    d3dpp.BackBufferFormat = d3ddm.Format;
 -    d3dpp.SwapEffect       = D3DSWAPEFFECT_DISCARD;
 -    d3dpp.Flags            = D3DPRESENTFLAG_VIDEO;
 -
 -    hr = IDirect3D9_CreateDevice(priv->d3d9, adapter, D3DDEVTYPE_HAL, GetShellWindow(),
 -                                 D3DCREATE_SOFTWARE_VERTEXPROCESSING | D3DCREATE_MULTITHREADED | D3DCREATE_FPU_PRESERVE,
 -                                 &d3dpp, &priv->d3d9device);
 -    if (FAILED(hr)) {
 -        av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device\n");
 -        return AVERROR_UNKNOWN;
 +    if (dxva2_device_create9ex(ctx, adapter) < 0) {
 +        // Retry with "classic" d3d9
 +        err = dxva2_device_create9(ctx, adapter);
 +        if (err < 0)
 +            return err;
      }
  
      hr = createDeviceManager(&resetToken, &hwctx->devmgr);