avutil/integer: Fix integer overflow in av_mul_i()
[ffmpeg.git] / libavutil / hwcontext_dxva2.c
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include <windows.h>
20
21 #define DXVA2API_USE_BITFIELDS
22 #define COBJMACROS
23
24 #include <d3d9.h>
25 #include <dxva2api.h>
26 #include <initguid.h>
27
28 #include "avassert.h"
29 #include "common.h"
30 #include "hwcontext.h"
31 #include "hwcontext_dxva2.h"
32 #include "hwcontext_internal.h"
33 #include "imgutils.h"
34 #include "pixdesc.h"
35 #include "pixfmt.h"
36 #include "compat/w32dlfcn.h"
37
38 typedef IDirect3D9* WINAPI pDirect3DCreate9(UINT);
39 typedef HRESULT WINAPI pDirect3DCreate9Ex(UINT, IDirect3D9Ex **);
40 typedef HRESULT WINAPI pCreateDeviceManager9(UINT *, IDirect3DDeviceManager9 **);
41
42 #define FF_D3DCREATE_FLAGS (D3DCREATE_SOFTWARE_VERTEXPROCESSING | \
43                             D3DCREATE_MULTITHREADED | \
44                             D3DCREATE_FPU_PRESERVE)
45
46 static const D3DPRESENT_PARAMETERS dxva2_present_params = {
47     .Windowed         = TRUE,
48     .BackBufferWidth  = 640,
49     .BackBufferHeight = 480,
50     .BackBufferCount  = 0,
51     .SwapEffect       = D3DSWAPEFFECT_DISCARD,
52     .Flags            = D3DPRESENTFLAG_VIDEO,
53 };
54
55 typedef struct DXVA2Mapping {
56     uint32_t palette_dummy[256];
57 } DXVA2Mapping;
58
59 typedef struct DXVA2FramesContext {
60     IDirect3DSurface9 **surfaces_internal;
61     int              nb_surfaces_used;
62
63     HANDLE  device_handle;
64     IDirectXVideoAccelerationService *service;
65
66     D3DFORMAT format;
67 } DXVA2FramesContext;
68
69 typedef struct DXVA2DevicePriv {
70     HMODULE d3dlib;
71     HMODULE dxva2lib;
72
73     HANDLE device_handle;
74
75     IDirect3D9       *d3d9;
76     IDirect3DDevice9 *d3d9device;
77 } DXVA2DevicePriv;
78
79 static const struct {
80     D3DFORMAT d3d_format;
81     enum AVPixelFormat pix_fmt;
82 } supported_formats[] = {
83     { MKTAG('N', 'V', '1', '2'), AV_PIX_FMT_NV12 },
84     { MKTAG('P', '0', '1', '0'), AV_PIX_FMT_P010 },
85     { D3DFMT_P8,                 AV_PIX_FMT_PAL8 },
86 };
87
88 DEFINE_GUID(video_decoder_service,   0xfc51a551, 0xd5e7, 0x11d9, 0xaf, 0x55, 0x00, 0x05, 0x4e, 0x43, 0xff, 0x02);
89 DEFINE_GUID(video_processor_service, 0xfc51a552, 0xd5e7, 0x11d9, 0xaf, 0x55, 0x00, 0x05, 0x4e, 0x43, 0xff, 0x02);
90
91 static void dxva2_frames_uninit(AVHWFramesContext *ctx)
92 {
93     AVDXVA2DeviceContext *device_hwctx = ctx->device_ctx->hwctx;
94     AVDXVA2FramesContext *frames_hwctx = ctx->hwctx;
95     DXVA2FramesContext *s = ctx->internal->priv;
96     int i;
97
98     if (frames_hwctx->decoder_to_release)
99         IDirectXVideoDecoder_Release(frames_hwctx->decoder_to_release);
100
101     if (s->surfaces_internal) {
102         for (i = 0; i < frames_hwctx->nb_surfaces; i++) {
103             if (s->surfaces_internal[i])
104                 IDirect3DSurface9_Release(s->surfaces_internal[i]);
105         }
106     }
107     av_freep(&s->surfaces_internal);
108
109     if (s->service) {
110         IDirectXVideoAccelerationService_Release(s->service);
111         s->service = NULL;
112     }
113
114     if (s->device_handle != INVALID_HANDLE_VALUE) {
115         IDirect3DDeviceManager9_CloseDeviceHandle(device_hwctx->devmgr, s->device_handle);
116         s->device_handle = INVALID_HANDLE_VALUE;
117     }
118 }
119
120 static void dxva2_pool_release_dummy(void *opaque, uint8_t *data)
121 {
122     // important not to free anything here--data is a surface object
123     // associated with the call to CreateSurface(), and these surfaces are
124     // released in dxva2_frames_uninit()
125 }
126
127 static AVBufferRef *dxva2_pool_alloc(void *opaque, int size)
128 {
129     AVHWFramesContext      *ctx = (AVHWFramesContext*)opaque;
130     DXVA2FramesContext       *s = ctx->internal->priv;
131     AVDXVA2FramesContext *hwctx = ctx->hwctx;
132
133     if (s->nb_surfaces_used < hwctx->nb_surfaces) {
134         s->nb_surfaces_used++;
135         return av_buffer_create((uint8_t*)s->surfaces_internal[s->nb_surfaces_used - 1],
136                                 sizeof(*hwctx->surfaces), dxva2_pool_release_dummy, 0, 0);
137     }
138
139     return NULL;
140 }
141
142 static int dxva2_init_pool(AVHWFramesContext *ctx)
143 {
144     AVDXVA2FramesContext *frames_hwctx = ctx->hwctx;
145     AVDXVA2DeviceContext *device_hwctx = ctx->device_ctx->hwctx;
146     DXVA2FramesContext              *s = ctx->internal->priv;
147     int decode = (frames_hwctx->surface_type == DXVA2_VideoDecoderRenderTarget);
148
149     int i;
150     HRESULT hr;
151
152     if (ctx->initial_pool_size <= 0)
153         return 0;
154
155     hr = IDirect3DDeviceManager9_OpenDeviceHandle(device_hwctx->devmgr, &s->device_handle);
156     if (FAILED(hr)) {
157         av_log(ctx, AV_LOG_ERROR, "Failed to open device handle\n");
158         return AVERROR_UNKNOWN;
159     }
160
161     hr = IDirect3DDeviceManager9_GetVideoService(device_hwctx->devmgr,
162                                                  s->device_handle,
163                                                  decode ? &video_decoder_service : &video_processor_service,
164                                                  (void **)&s->service);
165     if (FAILED(hr)) {
166         av_log(ctx, AV_LOG_ERROR, "Failed to create the video service\n");
167         return AVERROR_UNKNOWN;
168     }
169
170     for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
171         if (ctx->sw_format == supported_formats[i].pix_fmt) {
172             s->format = supported_formats[i].d3d_format;
173             break;
174         }
175     }
176     if (i == FF_ARRAY_ELEMS(supported_formats)) {
177         av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n",
178                av_get_pix_fmt_name(ctx->sw_format));
179         return AVERROR(EINVAL);
180     }
181
182     s->surfaces_internal = av_mallocz_array(ctx->initial_pool_size,
183                                             sizeof(*s->surfaces_internal));
184     if (!s->surfaces_internal)
185         return AVERROR(ENOMEM);
186
187     hr = IDirectXVideoAccelerationService_CreateSurface(s->service,
188                                                         ctx->width, ctx->height,
189                                                         ctx->initial_pool_size - 1,
190                                                         s->format, D3DPOOL_DEFAULT, 0,
191                                                         frames_hwctx->surface_type,
192                                                         s->surfaces_internal, NULL);
193     if (FAILED(hr)) {
194         av_log(ctx, AV_LOG_ERROR, "Could not create the surfaces\n");
195         return AVERROR_UNKNOWN;
196     }
197
198     ctx->internal->pool_internal = av_buffer_pool_init2(sizeof(*s->surfaces_internal),
199                                                         ctx, dxva2_pool_alloc, NULL);
200     if (!ctx->internal->pool_internal)
201         return AVERROR(ENOMEM);
202
203     frames_hwctx->surfaces    = s->surfaces_internal;
204     frames_hwctx->nb_surfaces = ctx->initial_pool_size;
205
206     return 0;
207 }
208
209 static int dxva2_frames_init(AVHWFramesContext *ctx)
210 {
211     AVDXVA2FramesContext *hwctx = ctx->hwctx;
212     DXVA2FramesContext       *s = ctx->internal->priv;
213     int ret;
214
215     if (hwctx->surface_type != DXVA2_VideoDecoderRenderTarget &&
216         hwctx->surface_type != DXVA2_VideoProcessorRenderTarget) {
217         av_log(ctx, AV_LOG_ERROR, "Unknown surface type: %lu\n",
218                hwctx->surface_type);
219         return AVERROR(EINVAL);
220     }
221
222     s->device_handle = INVALID_HANDLE_VALUE;
223
224     /* init the frame pool if the caller didn't provide one */
225     if (!ctx->pool) {
226         ret = dxva2_init_pool(ctx);
227         if (ret < 0) {
228             av_log(ctx, AV_LOG_ERROR, "Error creating an internal frame pool\n");
229             return ret;
230         }
231     }
232
233     return 0;
234 }
235
236 static int dxva2_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
237 {
238     frame->buf[0] = av_buffer_pool_get(ctx->pool);
239     if (!frame->buf[0])
240         return AVERROR(ENOMEM);
241
242     frame->data[3] = frame->buf[0]->data;
243     frame->format  = AV_PIX_FMT_DXVA2_VLD;
244     frame->width   = ctx->width;
245     frame->height  = ctx->height;
246
247     return 0;
248 }
249
250 static int dxva2_transfer_get_formats(AVHWFramesContext *ctx,
251                                       enum AVHWFrameTransferDirection dir,
252                                       enum AVPixelFormat **formats)
253 {
254     enum AVPixelFormat *fmts;
255
256     fmts = av_malloc_array(2, sizeof(*fmts));
257     if (!fmts)
258         return AVERROR(ENOMEM);
259
260     fmts[0] = ctx->sw_format;
261     fmts[1] = AV_PIX_FMT_NONE;
262
263     *formats = fmts;
264
265     return 0;
266 }
267
268 static void dxva2_unmap_frame(AVHWFramesContext *ctx, HWMapDescriptor *hwmap)
269 {
270     IDirect3DSurface9 *surface = (IDirect3DSurface9*)hwmap->source->data[3];
271     IDirect3DSurface9_UnlockRect(surface);
272     av_freep(&hwmap->priv);
273 }
274
275 static int dxva2_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src,
276                            int flags)
277 {
278     IDirect3DSurface9 *surface = (IDirect3DSurface9*)src->data[3];
279     DXVA2Mapping      *map;
280     D3DSURFACE_DESC    surfaceDesc;
281     D3DLOCKED_RECT     LockedRect;
282     HRESULT            hr;
283     int i, err, nb_planes;
284     int lock_flags = 0;
285
286     nb_planes = av_pix_fmt_count_planes(dst->format);
287
288     hr = IDirect3DSurface9_GetDesc(surface, &surfaceDesc);
289     if (FAILED(hr)) {
290         av_log(ctx, AV_LOG_ERROR, "Error getting a surface description\n");
291         return AVERROR_UNKNOWN;
292     }
293
294     if (!(flags & AV_HWFRAME_MAP_WRITE))
295         lock_flags |= D3DLOCK_READONLY;
296     if (flags & AV_HWFRAME_MAP_OVERWRITE)
297         lock_flags |= D3DLOCK_DISCARD;
298
299     hr = IDirect3DSurface9_LockRect(surface, &LockedRect, NULL, lock_flags);
300     if (FAILED(hr)) {
301         av_log(ctx, AV_LOG_ERROR, "Unable to lock DXVA2 surface\n");
302         return AVERROR_UNKNOWN;
303     }
304
305     map = av_mallocz(sizeof(*map));
306     if (!map) {
307         err = AVERROR(ENOMEM);
308         goto fail;
309     }
310
311     err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src,
312                                 dxva2_unmap_frame, map);
313     if (err < 0) {
314         av_freep(&map);
315         goto fail;
316     }
317
318     for (i = 0; i < nb_planes; i++)
319         dst->linesize[i] = LockedRect.Pitch;
320
321     av_image_fill_pointers(dst->data, dst->format, surfaceDesc.Height,
322                            (uint8_t*)LockedRect.pBits, dst->linesize);
323
324     if (dst->format == AV_PIX_FMT_PAL8)
325         dst->data[1] = (uint8_t*)map->palette_dummy;
326
327     return 0;
328 fail:
329     IDirect3DSurface9_UnlockRect(surface);
330     return err;
331 }
332
333 static int dxva2_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
334                                   const AVFrame *src)
335 {
336     AVFrame *map;
337     int ret;
338
339     if (src->format != ctx->sw_format)
340         return AVERROR(ENOSYS);
341
342     map = av_frame_alloc();
343     if (!map)
344         return AVERROR(ENOMEM);
345     map->format = dst->format;
346
347     ret = dxva2_map_frame(ctx, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE);
348     if (ret < 0)
349         goto fail;
350
351     av_image_copy(map->data, map->linesize, src->data, src->linesize,
352                   ctx->sw_format, src->width, src->height);
353
354 fail:
355     av_frame_free(&map);
356     return ret;
357 }
358
359 static int dxva2_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
360                                     const AVFrame *src)
361 {
362     AVFrame *map;
363     ptrdiff_t src_linesize[4], dst_linesize[4];
364     int ret, i;
365
366     if (dst->format != ctx->sw_format)
367         return AVERROR(ENOSYS);
368
369     map = av_frame_alloc();
370     if (!map)
371         return AVERROR(ENOMEM);
372     map->format = dst->format;
373
374     ret = dxva2_map_frame(ctx, map, src, AV_HWFRAME_MAP_READ);
375     if (ret < 0)
376         goto fail;
377
378     for (i = 0; i < 4; i++) {
379         dst_linesize[i] = dst->linesize[i];
380         src_linesize[i] = map->linesize[i];
381     }
382     av_image_copy_uc_from(dst->data, dst_linesize, map->data, src_linesize,
383                           ctx->sw_format, src->width, src->height);
384 fail:
385     av_frame_free(&map);
386     return ret;
387 }
388
389 static int dxva2_map_from(AVHWFramesContext *ctx,
390                           AVFrame *dst, const AVFrame *src, int flags)
391 {
392     int err;
393
394     if (dst->format != AV_PIX_FMT_NONE && dst->format != ctx->sw_format)
395         return AVERROR(ENOSYS);
396     dst->format = ctx->sw_format;
397
398     err = dxva2_map_frame(ctx, dst, src, flags);
399     if (err < 0)
400         return err;
401
402     err = av_frame_copy_props(dst, src);
403     if (err < 0)
404         return err;
405
406     return 0;
407 }
408
409 static void dxva2_device_free(AVHWDeviceContext *ctx)
410 {
411     AVDXVA2DeviceContext *hwctx = ctx->hwctx;
412     DXVA2DevicePriv       *priv = ctx->user_opaque;
413
414     if (hwctx->devmgr && priv->device_handle != INVALID_HANDLE_VALUE)
415         IDirect3DDeviceManager9_CloseDeviceHandle(hwctx->devmgr, priv->device_handle);
416
417     if (hwctx->devmgr)
418         IDirect3DDeviceManager9_Release(hwctx->devmgr);
419
420     if (priv->d3d9device)
421         IDirect3DDevice9_Release(priv->d3d9device);
422
423     if (priv->d3d9)
424         IDirect3D9_Release(priv->d3d9);
425
426     if (priv->d3dlib)
427         dlclose(priv->d3dlib);
428
429     if (priv->dxva2lib)
430         dlclose(priv->dxva2lib);
431
432     av_freep(&ctx->user_opaque);
433 }
434
435 static int dxva2_device_create9(AVHWDeviceContext *ctx, UINT adapter)
436 {
437     DXVA2DevicePriv *priv = ctx->user_opaque;
438     D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
439     D3DDISPLAYMODE d3ddm;
440     HRESULT hr;
441     pDirect3DCreate9 *createD3D = (pDirect3DCreate9 *)dlsym(priv->d3dlib, "Direct3DCreate9");
442     if (!createD3D) {
443         av_log(ctx, AV_LOG_ERROR, "Failed to locate Direct3DCreate9\n");
444         return AVERROR_UNKNOWN;
445     }
446
447     priv->d3d9 = createD3D(D3D_SDK_VERSION);
448     if (!priv->d3d9) {
449         av_log(ctx, AV_LOG_ERROR, "Failed to create IDirect3D object\n");
450         return AVERROR_UNKNOWN;
451     }
452
453     IDirect3D9_GetAdapterDisplayMode(priv->d3d9, adapter, &d3ddm);
454
455     d3dpp.BackBufferFormat = d3ddm.Format;
456
457     hr = IDirect3D9_CreateDevice(priv->d3d9, adapter, D3DDEVTYPE_HAL, GetDesktopWindow(),
458                                  FF_D3DCREATE_FLAGS,
459                                  &d3dpp, &priv->d3d9device);
460     if (FAILED(hr)) {
461         av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device\n");
462         return AVERROR_UNKNOWN;
463     }
464
465     return 0;
466 }
467
468 static int dxva2_device_create9ex(AVHWDeviceContext *ctx, UINT adapter)
469 {
470     DXVA2DevicePriv *priv = ctx->user_opaque;
471     D3DPRESENT_PARAMETERS d3dpp = dxva2_present_params;
472     D3DDISPLAYMODEEX modeex = {0};
473     IDirect3D9Ex *d3d9ex = NULL;
474     IDirect3DDevice9Ex *exdev = NULL;
475     HRESULT hr;
476     pDirect3DCreate9Ex *createD3DEx = (pDirect3DCreate9Ex *)dlsym(priv->d3dlib, "Direct3DCreate9Ex");
477     if (!createD3DEx)
478         return AVERROR(ENOSYS);
479
480     hr = createD3DEx(D3D_SDK_VERSION, &d3d9ex);
481     if (FAILED(hr))
482         return AVERROR_UNKNOWN;
483
484     modeex.Size = sizeof(D3DDISPLAYMODEEX);
485     hr = IDirect3D9Ex_GetAdapterDisplayModeEx(d3d9ex, adapter, &modeex, NULL);
486     if (FAILED(hr)) {
487         IDirect3D9Ex_Release(d3d9ex);
488         return AVERROR_UNKNOWN;
489     }
490
491     d3dpp.BackBufferFormat = modeex.Format;
492
493     hr = IDirect3D9Ex_CreateDeviceEx(d3d9ex, adapter, D3DDEVTYPE_HAL, GetDesktopWindow(),
494                                      FF_D3DCREATE_FLAGS,
495                                      &d3dpp, NULL, &exdev);
496     if (FAILED(hr)) {
497         IDirect3D9Ex_Release(d3d9ex);
498         return AVERROR_UNKNOWN;
499     }
500
501     av_log(ctx, AV_LOG_VERBOSE, "Using D3D9Ex device.\n");
502     priv->d3d9 = (IDirect3D9 *)d3d9ex;
503     priv->d3d9device = (IDirect3DDevice9 *)exdev;
504     return 0;
505 }
506
507 static int dxva2_device_create(AVHWDeviceContext *ctx, const char *device,
508                                AVDictionary *opts, int flags)
509 {
510     AVDXVA2DeviceContext *hwctx = ctx->hwctx;
511     DXVA2DevicePriv *priv;
512     pCreateDeviceManager9 *createDeviceManager = NULL;
513     unsigned resetToken = 0;
514     UINT adapter = D3DADAPTER_DEFAULT;
515     HRESULT hr;
516     int err;
517
518     if (device)
519         adapter = atoi(device);
520
521     priv = av_mallocz(sizeof(*priv));
522     if (!priv)
523         return AVERROR(ENOMEM);
524
525     ctx->user_opaque = priv;
526     ctx->free        = dxva2_device_free;
527
528     priv->device_handle = INVALID_HANDLE_VALUE;
529
530     priv->d3dlib = dlopen("d3d9.dll", 0);
531     if (!priv->d3dlib) {
532         av_log(ctx, AV_LOG_ERROR, "Failed to load D3D9 library\n");
533         return AVERROR_UNKNOWN;
534     }
535     priv->dxva2lib = dlopen("dxva2.dll", 0);
536     if (!priv->dxva2lib) {
537         av_log(ctx, AV_LOG_ERROR, "Failed to load DXVA2 library\n");
538         return AVERROR_UNKNOWN;
539     }
540
541     createDeviceManager = (pCreateDeviceManager9 *)dlsym(priv->dxva2lib,
542                                                          "DXVA2CreateDirect3DDeviceManager9");
543     if (!createDeviceManager) {
544         av_log(ctx, AV_LOG_ERROR, "Failed to locate DXVA2CreateDirect3DDeviceManager9\n");
545         return AVERROR_UNKNOWN;
546     }
547
548     if (dxva2_device_create9ex(ctx, adapter) < 0) {
549         // Retry with "classic" d3d9
550         err = dxva2_device_create9(ctx, adapter);
551         if (err < 0)
552             return err;
553     }
554
555     hr = createDeviceManager(&resetToken, &hwctx->devmgr);
556     if (FAILED(hr)) {
557         av_log(ctx, AV_LOG_ERROR, "Failed to create Direct3D device manager\n");
558         return AVERROR_UNKNOWN;
559     }
560
561     hr = IDirect3DDeviceManager9_ResetDevice(hwctx->devmgr, priv->d3d9device, resetToken);
562     if (FAILED(hr)) {
563         av_log(ctx, AV_LOG_ERROR, "Failed to bind Direct3D device to device manager\n");
564         return AVERROR_UNKNOWN;
565     }
566
567     hr = IDirect3DDeviceManager9_OpenDeviceHandle(hwctx->devmgr, &priv->device_handle);
568     if (FAILED(hr)) {
569         av_log(ctx, AV_LOG_ERROR, "Failed to open device handle\n");
570         return AVERROR_UNKNOWN;
571     }
572
573     return 0;
574 }
575
576 const HWContextType ff_hwcontext_type_dxva2 = {
577     .type                 = AV_HWDEVICE_TYPE_DXVA2,
578     .name                 = "DXVA2",
579
580     .device_hwctx_size    = sizeof(AVDXVA2DeviceContext),
581     .frames_hwctx_size    = sizeof(AVDXVA2FramesContext),
582     .frames_priv_size     = sizeof(DXVA2FramesContext),
583
584     .device_create        = dxva2_device_create,
585     .frames_init          = dxva2_frames_init,
586     .frames_uninit        = dxva2_frames_uninit,
587     .frames_get_buffer    = dxva2_get_buffer,
588     .transfer_get_formats = dxva2_transfer_get_formats,
589     .transfer_data_to     = dxva2_transfer_data_to,
590     .transfer_data_from   = dxva2_transfer_data_from,
591     .map_from             = dxva2_map_from,
592
593     .pix_fmts             = (const enum AVPixelFormat[]){ AV_PIX_FMT_DXVA2_VLD, AV_PIX_FMT_NONE },
594 };