MSW: fix assertion failure on Open() error paths (fix #3889)
[vlc.git] / modules / video_output / msw / direct3d.c
1 /*****************************************************************************
2  * direct3d.c: Windows Direct3D video output module
3  *****************************************************************************
4  * Copyright (C) 2006-2009 the VideoLAN team
5  *$Id$
6  *
7  * Authors: Damien Fouilleul <damienf@videolan.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * Preamble:
26  *
27  * This plugin will use YUV surface if supported, using YUV will result in
28  * the best video quality (hardware filering when rescaling the picture)
29  * and the fastest display as it requires less processing.
30  *
31  * If YUV overlay is not supported this plugin will use RGB offscreen video
32  * surfaces that will be blitted onto the primary surface (display) to
33  * effectively display the pictures.
34  *
35  *****************************************************************************/
36 #ifdef HAVE_CONFIG_H
37 # include "config.h"
38 #endif
39
40 #include <vlc_common.h>
41 #include <vlc_plugin.h>
42 #include <vlc_playlist.h>
43 #include <vlc_vout_display.h>
44
45 #include <windows.h>
46 #include <d3d9.h>
47
48 #include "common.h"
49
50 /*****************************************************************************
51  * Module descriptor
52  *****************************************************************************/
53 static int  OpenVideoXP(vlc_object_t *);
54 static int  OpenVideoVista(vlc_object_t *);
55 static void Close(vlc_object_t *);
56
57 #define DESKTOP_TEXT N_("Enable desktop mode ")
58 #define DESKTOP_LONGTEXT N_(\
59     "The desktop mode allows you to display the video on the desktop.")
60
61 #define D3D_HELP N_("Recommended video output for Windows Vista and later versions")
62
63 vlc_module_begin ()
64     set_shortname("Direct3D")
65     set_description(N_("Direct3D video output"))
66     set_help(D3D_HELP)
67     set_category(CAT_VIDEO)
68     set_subcategory(SUBCAT_VIDEO_VOUT)
69
70     add_bool("direct3d-desktop", false, NULL, DESKTOP_TEXT, DESKTOP_LONGTEXT, true)
71
72     set_capability("vout display", 150)
73     add_shortcut("direct3d")
74     set_callbacks(OpenVideoVista, Close)
75
76     add_submodule()
77         set_description(N_("Direct3D video output (XP)"))
78         set_capability("vout display", 70)
79         add_shortcut("direct3d_xp")
80         set_callbacks(OpenVideoXP, Close)
81
82 vlc_module_end ()
83
84 /*****************************************************************************
85  * Local prototypes.
86  *****************************************************************************/
87 struct picture_sys_t
88 {
89     LPDIRECT3DSURFACE9 surface;
90 };
91
92 static int  Open(vlc_object_t *);
93
94 static picture_pool_t *Pool  (vout_display_t *, unsigned);
95 static void           Prepare(vout_display_t *, picture_t *);
96 static void           Display(vout_display_t *, picture_t *);
97 static int            Control(vout_display_t *, int, va_list);
98 static void           Manage (vout_display_t *);
99
100 static int  Direct3DCreate (vout_display_t *);
101 static int  Direct3DReset  (vout_display_t *);
102 static void Direct3DDestroy(vout_display_t *);
103
104 static int  Direct3DOpen (vout_display_t *, video_format_t *);
105 static void Direct3DClose(vout_display_t *);
106
107 static void Direct3DRenderScene(vout_display_t *vd, LPDIRECT3DSURFACE9 surface);
108
109 /* */
110 static int DesktopCallback(vlc_object_t *, char const *, vlc_value_t, vlc_value_t, void *);
111
112 /**
113  * It creates a Direct3D vout display.
114  */
115 static int Open(vlc_object_t *object)
116 {
117     vout_display_t *vd = (vout_display_t *)object;
118     vout_display_sys_t *sys;
119
120     /* Allocate structure */
121     vd->sys = sys = calloc(1, sizeof(vout_display_sys_t));
122     if (!sys)
123         return VLC_ENOMEM;
124
125     if (Direct3DCreate(vd)) {
126         msg_Err(vd, "Direct3D could not be initialized");
127         Direct3DDestroy(vd);
128         free(sys);
129         return VLC_EGENERIC;
130     }
131
132     sys->use_desktop = var_CreateGetBool(vd, "direct3d-desktop");
133     sys->reset_device = false;
134     sys->reset_device = false;
135     sys->allow_hw_yuv = var_CreateGetBool(vd, "directx-hw-yuv");
136     sys->desktop_save.is_fullscreen = vd->cfg->is_fullscreen;
137     sys->desktop_save.is_on_top     = false;
138     sys->desktop_save.win.left      = var_InheritInteger(vd, "video-x");
139     sys->desktop_save.win.right     = vd->cfg->display.width;
140     sys->desktop_save.win.top       = var_InheritInteger(vd, "video-y");
141     sys->desktop_save.win.bottom    = vd->cfg->display.height;
142
143     if (CommonInit(vd))
144         goto error;
145
146     /* */
147     video_format_t fmt;
148     if (Direct3DOpen(vd, &fmt)) {
149         msg_Err(vd, "Direct3D could not be opened");
150         goto error;
151     }
152
153     /* */
154     vout_display_info_t info = vd->info;
155     info.is_slow = true;
156     info.has_double_click = true;
157     info.has_hide_mouse = false;
158     info.has_pictures_invalid = true;
159     info.has_event_thread = true;
160
161     /* Interaction */
162     vlc_mutex_init(&sys->lock);
163     sys->ch_desktop = false;
164     sys->desktop_requested = sys->use_desktop;
165
166     vlc_value_t val;
167     val.psz_string = _("Desktop");
168     var_Change(vd, "direct3d-desktop", VLC_VAR_SETTEXT, &val, NULL);
169     var_AddCallback(vd, "direct3d-desktop", DesktopCallback, NULL);
170
171     /* Setup vout_display now that everything is fine */
172     vd->fmt  = fmt;
173     vd->info = info;
174
175     vd->pool    = Pool;
176     vd->prepare = Prepare;
177     vd->display = Display;
178     vd->control = Control;
179     vd->manage  = Manage;
180
181     /* Fix state in case of desktop mode */
182     if (sys->use_desktop && vd->cfg->is_fullscreen)
183         vout_display_SendEventFullscreen(vd, false);
184
185     return VLC_SUCCESS;
186 error:
187     Direct3DClose(vd);
188     CommonClean(vd);
189     Direct3DDestroy(vd);
190     free(vd->sys);
191     return VLC_EGENERIC;
192 }
193
194 static bool IsVistaOrAbove(void)
195 {
196     OSVERSIONINFO winVer;
197     winVer.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
198     return GetVersionEx(&winVer) && winVer.dwMajorVersion > 5;
199 }
200
201 static int OpenVideoXP(vlc_object_t *obj)
202 {
203     /* Windows XP or lower, make sure this module isn't the default */
204     return IsVistaOrAbove() ? VLC_EGENERIC : Open(obj);
205 }
206
207 static int OpenVideoVista(vlc_object_t *obj)
208 {
209     /* Windows Vista or above, make this module the default */
210     return IsVistaOrAbove() ? Open(obj) : VLC_EGENERIC;
211 }
212
213 /**
214  * It destroyes a Direct3D vout display.
215  */
216 static void Close(vlc_object_t *object)
217 {
218     vout_display_t * vd = (vout_display_t *)object;
219
220     var_DelCallback(vd, "direct3d-desktop", DesktopCallback, NULL);
221
222     Direct3DClose(vd);
223
224     CommonClean(vd);
225
226     Direct3DDestroy(vd);
227
228     free(vd->sys);
229 }
230
231 /* */
232 static picture_pool_t *Pool(vout_display_t *vd, unsigned count)
233 {
234     VLC_UNUSED(count);
235     return vd->sys->pool;
236 }
237
238 static int  Direct3DLockSurface(picture_t *);
239 static void Direct3DUnlockSurface(picture_t *);
240
241 static void Prepare(vout_display_t *vd, picture_t *picture)
242 {
243     LPDIRECT3DSURFACE9 surface = picture->p_sys->surface;
244 #if 0
245     picture_Release(picture);
246     Direct3DRenderScene(vd, surface);
247 #else
248     /* FIXME it is a bit ugly, we need the surface to be unlocked for
249      * rendering.
250      *  The clean way would be to release the picture (and ensure that
251      * the vout doesn't keep a reference). But because of the vout
252      * wrapper, we can't */
253
254     Direct3DUnlockSurface(picture);
255
256     Direct3DRenderScene(vd, surface);
257
258     Direct3DLockSurface(picture);
259 #endif
260 }
261
262 static void Display(vout_display_t *vd, picture_t *picture)
263 {
264     vout_display_sys_t *sys = vd->sys;
265     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
266
267     // Present the back buffer contents to the display
268     // No stretching should happen here !
269     const RECT src = sys->rect_dest_clipped;
270     const RECT dst = sys->rect_dest_clipped;
271     HRESULT hr = IDirect3DDevice9_Present(d3ddev, &src, &dst, NULL, NULL);
272     if (FAILED(hr)) {
273         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
274     }
275 #if 0
276     VLC_UNUSED(picture);
277 #else
278     /* XXX See Prepare() */
279     picture_Release(picture);
280 #endif
281
282     CommonDisplay(vd);
283 }
284 static int ControlResetDevice(vout_display_t *vd)
285 {
286     return Direct3DReset(vd);
287 }
288 static int ControlReopenDevice(vout_display_t *vd)
289 {
290     vout_display_sys_t *sys = vd->sys;
291
292     if (!sys->use_desktop) {
293         /* Save non-desktop state */
294         sys->desktop_save.is_fullscreen = vd->cfg->is_fullscreen;
295         sys->desktop_save.is_on_top     = sys->is_on_top;
296
297         WINDOWPLACEMENT wp = { .length = sizeof(wp), };
298         GetWindowPlacement(sys->hparent ? sys->hparent : sys->hwnd, &wp);
299         sys->desktop_save.win = wp.rcNormalPosition;
300     }
301
302     /* */
303     Direct3DClose(vd);
304     EventThreadStop(sys->event);
305
306     /* */
307     vlc_mutex_lock(&sys->lock);
308     sys->use_desktop = sys->desktop_requested;
309     sys->ch_desktop = false;
310     vlc_mutex_unlock(&sys->lock);
311
312     /* */
313     event_cfg_t cfg;
314     memset(&cfg, 0, sizeof(cfg));
315     cfg.use_desktop = sys->use_desktop;
316     if (!sys->use_desktop) {
317         cfg.win.type   = VOUT_WINDOW_TYPE_HWND;
318         cfg.win.x      = sys->desktop_save.win.left;
319         cfg.win.y      = sys->desktop_save.win.top;
320         cfg.win.width  = sys->desktop_save.win.right  - sys->desktop_save.win.left;
321         cfg.win.height = sys->desktop_save.win.bottom - sys->desktop_save.win.top;
322     }
323
324     event_hwnd_t hwnd;
325     if (EventThreadStart(sys->event, &hwnd, &cfg)) {
326         msg_Err(vd, "Failed to restart event thread");
327         return VLC_EGENERIC;
328     }
329     sys->parent_window = hwnd.parent_window;
330     sys->hparent       = hwnd.hparent;
331     sys->hwnd          = hwnd.hwnd;
332     sys->hvideownd     = hwnd.hvideownd;
333     sys->hfswnd        = hwnd.hfswnd;
334     SetRectEmpty(&sys->rect_parent);
335
336     /* */
337     video_format_t fmt;
338     if (Direct3DOpen(vd, &fmt)) {
339         CommonClean(vd);
340         msg_Err(vd, "Failed to reopen device");
341         return VLC_EGENERIC;
342     }
343     vd->fmt = fmt;
344     sys->is_first_display = true;
345
346     if (sys->use_desktop) {
347         /* Disable fullscreen/on_top while using desktop */
348         if (sys->desktop_save.is_fullscreen)
349             vout_display_SendEventFullscreen(vd, false);
350         if (sys->desktop_save.is_on_top)
351             vout_display_SendWindowState(vd, VOUT_WINDOW_STATE_NORMAL);
352     } else {
353         /* Restore fullscreen/on_top */
354         if (sys->desktop_save.is_fullscreen)
355             vout_display_SendEventFullscreen(vd, true);
356         if (sys->desktop_save.is_on_top)
357             vout_display_SendWindowState(vd, VOUT_WINDOW_STATE_ABOVE);
358     }
359     return VLC_SUCCESS;
360 }
361 static int Control(vout_display_t *vd, int query, va_list args)
362 {
363     vout_display_sys_t *sys = vd->sys;
364
365     switch (query) {
366     case VOUT_DISPLAY_RESET_PICTURES:
367         /* FIXME what to do here in case of failure */
368         if (sys->reset_device) {
369             if (ControlResetDevice(vd)) {
370                 msg_Err(vd, "Failed to reset device");
371                 return VLC_EGENERIC;
372             }
373             sys->reset_device = false;
374         } else if(sys->reopen_device) {
375             if (ControlReopenDevice(vd)) {
376                 msg_Err(vd, "Failed to reopen device");
377                 return VLC_EGENERIC;
378             }
379             sys->reopen_device = false;
380         }
381         return VLC_SUCCESS;
382     default:
383         return CommonControl(vd, query, args);
384     }
385 }
386 static void Manage (vout_display_t *vd)
387 {
388     vout_display_sys_t *sys = vd->sys;
389
390     CommonManage(vd);
391
392     /* Desktop mode change */
393     vlc_mutex_lock(&sys->lock);
394     const bool ch_desktop = sys->ch_desktop;
395     sys->ch_desktop = false;
396     vlc_mutex_unlock(&sys->lock);
397
398     if (ch_desktop) {
399         sys->reopen_device = true;
400         vout_display_SendEventPicturesInvalid(vd);
401     }
402
403 #if 0
404     /*
405      * Position Change
406      */
407     if (sys->changes & DX_POSITION_CHANGE) {
408 #if 0 /* need that when bicubic filter is available */
409         RECT rect;
410         UINT width, height;
411
412         GetClientRect(p_sys->hvideownd, &rect);
413         width  = rect.right-rect.left;
414         height = rect.bottom-rect.top;
415
416         if (width != p_sys->d3dpp.BackBufferWidth || height != p_sys->d3dpp.BackBufferHeight)
417         {
418             msg_Dbg(vd, "resizing device back buffers to (%lux%lu)", width, height);
419             // need to reset D3D device to resize back buffer
420             if (VLC_SUCCESS != Direct3DResetDevice(vd, width, height))
421                 return VLC_EGENERIC;
422         }
423 #endif
424         sys->changes &= ~DX_POSITION_CHANGE;
425     }
426 #endif
427 }
428
429 /**
430  * It initializes an instance of Direct3D9
431  */
432 static int Direct3DCreate(vout_display_t *vd)
433 {
434     vout_display_sys_t *sys = vd->sys;
435
436     sys->hd3d9_dll = LoadLibrary(TEXT("D3D9.DLL"));
437     if (!sys->hd3d9_dll) {
438         msg_Warn(vd, "cannot load d3d9.dll, aborting");
439         return VLC_EGENERIC;
440     }
441
442     LPDIRECT3D9 (WINAPI *OurDirect3DCreate9)(UINT SDKVersion);
443     OurDirect3DCreate9 =
444         (void *)GetProcAddress(sys->hd3d9_dll, TEXT("Direct3DCreate9"));
445     if (!OurDirect3DCreate9) {
446         msg_Err(vd, "Cannot locate reference to Direct3DCreate9 ABI in DLL");
447         return VLC_EGENERIC;
448     }
449
450     /* Create the D3D object. */
451     LPDIRECT3D9 d3dobj = OurDirect3DCreate9(D3D_SDK_VERSION);
452     if (!d3dobj) {
453        msg_Err(vd, "Could not create Direct3D9 instance.");
454        return VLC_EGENERIC;
455     }
456     sys->d3dobj = d3dobj;
457
458     /*
459     ** Get device capabilities
460     */
461     D3DCAPS9 d3dCaps;
462     ZeroMemory(&d3dCaps, sizeof(d3dCaps));
463     HRESULT hr = IDirect3D9_GetDeviceCaps(d3dobj, D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, &d3dCaps);
464     if (FAILED(hr)) {
465        msg_Err(vd, "Could not read adapter capabilities. (hr=0x%lX)", hr);
466        return VLC_EGENERIC;
467     }
468     /* TODO: need to test device capabilities and select the right render function */
469
470     return VLC_SUCCESS;
471 }
472
473 /**
474  * It releases an instance of Direct3D9
475  */
476 static void Direct3DDestroy(vout_display_t *vd)
477 {
478     vout_display_sys_t *sys = vd->sys;
479
480     if (sys->d3dobj)
481        IDirect3D9_Release(sys->d3dobj);
482     if (sys->hd3d9_dll)
483         FreeLibrary(sys->hd3d9_dll);
484
485     sys->d3dobj = NULL;
486     sys->hd3d9_dll = NULL;
487 }
488
489
490 /**
491  * It setup vout_display_sys_t::d3dpp and vout_display_sys_t::rect_display
492  * from the default adapter.
493  */
494 static int Direct3DFillPresentationParameters(vout_display_t *vd)
495 {
496     vout_display_sys_t *sys = vd->sys;
497
498     /*
499     ** Get the current desktop display mode, so we can set up a back
500     ** buffer of the same format
501     */
502     D3DDISPLAYMODE d3ddm;
503     HRESULT hr = IDirect3D9_GetAdapterDisplayMode(sys->d3dobj,
504                                                   D3DADAPTER_DEFAULT, &d3ddm);
505     if (FAILED(hr)) {
506        msg_Err(vd, "Could not read adapter display mode. (hr=0x%lX)", hr);
507        return VLC_EGENERIC;
508     }
509
510     /* Set up the structure used to create the D3DDevice. */
511     D3DPRESENT_PARAMETERS *d3dpp = &vd->sys->d3dpp;
512     ZeroMemory(d3dpp, sizeof(D3DPRESENT_PARAMETERS));
513     d3dpp->Flags                  = D3DPRESENTFLAG_VIDEO;
514     d3dpp->Windowed               = TRUE;
515     d3dpp->hDeviceWindow          = vd->sys->hvideownd;
516     d3dpp->BackBufferWidth        = __MAX(GetSystemMetrics(SM_CXVIRTUALSCREEN),
517                                           d3ddm.Width);
518     d3dpp->BackBufferHeight       = __MAX(GetSystemMetrics(SM_CYVIRTUALSCREEN),
519                                           d3ddm.Height);
520     d3dpp->SwapEffect             = D3DSWAPEFFECT_COPY;
521     d3dpp->MultiSampleType        = D3DMULTISAMPLE_NONE;
522     d3dpp->PresentationInterval   = D3DPRESENT_INTERVAL_DEFAULT;
523     d3dpp->BackBufferFormat       = d3ddm.Format;
524     d3dpp->BackBufferCount        = 1;
525     d3dpp->EnableAutoDepthStencil = FALSE;
526
527     /* */
528     RECT *display = &vd->sys->rect_display;
529     display->left   = 0;
530     display->top    = 0;
531     display->right  = d3dpp->BackBufferWidth;
532     display->bottom = d3dpp->BackBufferHeight;
533
534     return VLC_SUCCESS;
535 }
536
537 /* */
538 static int  Direct3DCreateResources (vout_display_t *, video_format_t *);
539 static void Direct3DDestroyResources(vout_display_t *);
540
541 /**
542  * It creates a Direct3D device and the associated resources.
543  */
544 static int Direct3DOpen(vout_display_t *vd, video_format_t *fmt)
545 {
546     vout_display_sys_t *sys = vd->sys;
547     LPDIRECT3D9 d3dobj = sys->d3dobj;
548
549     if (Direct3DFillPresentationParameters(vd))
550         return VLC_EGENERIC;
551
552     // Create the D3DDevice
553     LPDIRECT3DDEVICE9 d3ddev;
554     HRESULT hr = IDirect3D9_CreateDevice(d3dobj, D3DADAPTER_DEFAULT,
555                                          D3DDEVTYPE_HAL, sys->hvideownd,
556                                          D3DCREATE_SOFTWARE_VERTEXPROCESSING|
557                                          D3DCREATE_MULTITHREADED,
558                                          &sys->d3dpp, &d3ddev);
559     if (FAILED(hr)) {
560        msg_Err(vd, "Could not create the D3D device! (hr=0x%lX)", hr);
561        return VLC_EGENERIC;
562     }
563     sys->d3ddev = d3ddev;
564
565     UpdateRects(vd, NULL, NULL, true);
566
567     if (Direct3DCreateResources(vd, fmt)) {
568         msg_Err(vd, "Failed to allocate resources");
569         return VLC_EGENERIC;
570     }
571
572     /* Change the window title bar text */
573     EventThreadUpdateTitle(sys->event, VOUT_TITLE " (Direct3D output)");
574
575     msg_Dbg(vd, "Direct3D device adapter successfully initialized");
576     return VLC_SUCCESS;
577 }
578
579 /**
580  * It releases the Direct3D9 device and its resources.
581  */
582 static void Direct3DClose(vout_display_t *vd)
583 {
584     vout_display_sys_t *sys = vd->sys;
585
586     Direct3DDestroyResources(vd);
587
588     if (sys->d3ddev)
589        IDirect3DDevice9_Release(sys->d3ddev);
590
591     sys->d3ddev = NULL;
592 }
593
594 /**
595  * It reset the Direct3D9 device and its resources.
596  */
597 static int Direct3DReset(vout_display_t *vd)
598 {
599     vout_display_sys_t *sys = vd->sys;
600     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
601
602     if (Direct3DFillPresentationParameters(vd))
603         return VLC_EGENERIC;
604
605     /* release all D3D objects */
606     Direct3DDestroyResources(vd);
607
608     /* */
609     HRESULT hr = IDirect3DDevice9_Reset(d3ddev, &sys->d3dpp);
610     if (FAILED(hr)) {
611         msg_Err(vd, "%s failed ! (hr=%08lX)", __FUNCTION__, hr);
612         return VLC_EGENERIC;
613     }
614
615     UpdateRects(vd, NULL, NULL, true);
616
617     /* re-create them */
618     if (Direct3DCreateResources(vd, &vd->fmt)) {
619         msg_Dbg(vd, "%s failed !", __FUNCTION__);
620         return VLC_EGENERIC;
621     }
622     return VLC_SUCCESS;
623 }
624
625 /* */
626 static int  Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt);
627 static void Direct3DDestroyPool(vout_display_t *vd);
628
629 static int  Direct3DCreateScene(vout_display_t *vd);
630 static void Direct3DDestroyScene(vout_display_t *vd);
631
632 /**
633  * It creates the picture and scene resources.
634  */
635 static int Direct3DCreateResources(vout_display_t *vd, video_format_t *fmt)
636 {
637     if (Direct3DCreatePool(vd, fmt)) {
638         msg_Err(vd, "Direct3D picture pool initialization failed");
639         return VLC_EGENERIC;
640     }
641     if (Direct3DCreateScene(vd)) {
642         msg_Err(vd, "Direct3D scene initialization failed !");
643         return VLC_EGENERIC;
644     }
645     return VLC_SUCCESS;
646 }
647 /**
648  * It destroys the picture and scene resources.
649  */
650 static void Direct3DDestroyResources(vout_display_t *vd)
651 {
652     Direct3DDestroyScene(vd);
653     Direct3DDestroyPool(vd);
654 }
655
656 /**
657  * It tests if the conversion from src to dst is supported.
658  */
659 static int Direct3DCheckConversion(vout_display_t *vd,
660                                    D3DFORMAT src, D3DFORMAT dst)
661 {
662     vout_display_sys_t *sys = vd->sys;
663     LPDIRECT3D9 d3dobj = sys->d3dobj;
664     HRESULT hr;
665
666     /* test whether device can create a surface of that format */
667     hr = IDirect3D9_CheckDeviceFormat(d3dobj, D3DADAPTER_DEFAULT,
668                                       D3DDEVTYPE_HAL, dst, 0,
669                                       D3DRTYPE_SURFACE, src);
670     if (SUCCEEDED(hr)) {
671         /* test whether device can perform color-conversion
672         ** from that format to target format
673         */
674         hr = IDirect3D9_CheckDeviceFormatConversion(d3dobj,
675                                                     D3DADAPTER_DEFAULT,
676                                                     D3DDEVTYPE_HAL,
677                                                     src, dst);
678     }
679     if (!SUCCEEDED(hr)) {
680         if (D3DERR_NOTAVAILABLE != hr)
681             msg_Err(vd, "Could not query adapter supported formats. (hr=0x%lX)", hr);
682         return VLC_EGENERIC;
683     }
684     return VLC_SUCCESS;
685 }
686
687 typedef struct
688 {
689     const char   *name;
690     D3DFORMAT    format;    /* D3D format */
691     vlc_fourcc_t fourcc;    /* VLC fourcc */
692     uint32_t     rmask;
693     uint32_t     gmask;
694     uint32_t     bmask;
695 } d3d_format_t;
696
697 static const d3d_format_t d3d_formats[] = {
698     /* YV12 is always used for planar 420, the planes are then swapped in Lock() */
699     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_YV12,  0,0,0 },
700     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_I420,  0,0,0 },
701     { "YV12",       MAKEFOURCC('Y','V','1','2'),    VLC_CODEC_J420,  0,0,0 },
702     { "UYVY",       D3DFMT_UYVY,    VLC_CODEC_UYVY,  0,0,0 },
703     { "YUY2",       D3DFMT_YUY2,    VLC_CODEC_YUYV,  0,0,0 },
704     { "X8R8G8B8",   D3DFMT_X8R8G8B8,VLC_CODEC_RGB32, 0xff0000, 0x00ff00, 0x0000ff },
705     { "A8R8G8B8",   D3DFMT_A8R8G8B8,VLC_CODEC_RGB32, 0xff0000, 0x00ff00, 0x0000ff },
706     { "8G8B8",      D3DFMT_R8G8B8,  VLC_CODEC_RGB24, 0xff0000, 0x00ff00, 0x0000ff },
707     { "R5G6B5",     D3DFMT_R5G6B5,  VLC_CODEC_RGB16, 0x1f<<11, 0x3f<<5,  0x1f<<0 },
708     { "X1R5G5B5",   D3DFMT_X1R5G5B5,VLC_CODEC_RGB15, 0x1f<<10, 0x1f<<5,  0x1f<<0 },
709
710     { NULL, 0, 0, 0,0,0}
711 };
712
713 /**
714  * It returns the format (closest to chroma) that can be converted to target */
715 static const d3d_format_t *Direct3DFindFormat(vout_display_t *vd, vlc_fourcc_t chroma, D3DFORMAT target)
716 {
717     vout_display_sys_t *sys = vd->sys;
718
719     for (unsigned pass = 0; pass < 2; pass++) {
720         const vlc_fourcc_t *list;
721
722         if (pass == 0 && sys->allow_hw_yuv && vlc_fourcc_IsYUV(chroma))
723             list = vlc_fourcc_GetYUVFallback(chroma);
724         else if (pass == 1)
725             list = vlc_fourcc_GetRGBFallback(chroma);
726         else
727             continue;
728
729         for (unsigned i = 0; list[i] != 0; i++) {
730             for (unsigned j = 0; d3d_formats[j].name; j++) {
731                 const d3d_format_t *format = &d3d_formats[j];
732
733                 if (format->fourcc != list[i])
734                     continue;
735
736                 msg_Warn(vd, "trying surface pixel format: %s",
737                          format->name);
738                 if (!Direct3DCheckConversion(vd, format->format, target)) {
739                     msg_Dbg(vd, "selected surface pixel format is %s",
740                             format->name);
741                     return format;
742                 }
743             }
744         }
745     }
746     return NULL;
747 }
748
749 /**
750  * It locks the surface associated to the picture and get the surface
751  * descriptor which amongst other things has the pointer to the picture
752  * data and its pitch.
753  */
754 static int Direct3DLockSurface(picture_t *picture)
755 {
756     /* Lock the surface to get a valid pointer to the picture buffer */
757     D3DLOCKED_RECT d3drect;
758     HRESULT hr = IDirect3DSurface9_LockRect(picture->p_sys->surface, &d3drect, NULL, 0);
759     if (FAILED(hr)) {
760         //msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
761         return VLC_EGENERIC;
762     }
763
764     /* fill in buffer info in first plane */
765     picture->p->p_pixels = d3drect.pBits;
766     picture->p->i_pitch  = d3drect.Pitch;
767
768     /*  Fill chroma planes for planar YUV */
769     if (picture->format.i_chroma == VLC_CODEC_I420 ||
770         picture->format.i_chroma == VLC_CODEC_J420 ||
771         picture->format.i_chroma == VLC_CODEC_YV12) {
772
773         for (int n = 1; n < picture->i_planes; n++) {
774             const plane_t *o = &picture->p[n-1];
775             plane_t *p = &picture->p[n];
776
777             p->p_pixels = o->p_pixels + o->i_lines * o->i_pitch;
778             p->i_pitch  = d3drect.Pitch / 2;
779         }
780         /* The d3d buffer is always allocated as YV12 */
781         if (vlc_fourcc_AreUVPlanesSwapped(picture->format.i_chroma, VLC_CODEC_YV12)) {
782             uint8_t *p_tmp = picture->p[1].p_pixels;
783             picture->p[1].p_pixels = picture->p[2].p_pixels;
784             picture->p[2].p_pixels = p_tmp;
785         }
786     }
787     return VLC_SUCCESS;
788 }
789 /**
790  * It unlocks the surface associated to the picture.
791  */
792 static void Direct3DUnlockSurface(picture_t *picture)
793 {
794     /* Unlock the Surface */
795     HRESULT hr = IDirect3DSurface9_UnlockRect(picture->p_sys->surface);
796     if (FAILED(hr)) {
797         //msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
798     }
799 }
800
801 /**
802  * It creates the pool of picture (only 1).
803  *
804  * Each picture has an associated offscreen surface in video memory
805  * depending on hardware capabilities the picture chroma will be as close
806  * as possible to the orginal render chroma to reduce CPU conversion overhead
807  * and delegate this work to video card GPU
808  */
809 static int Direct3DCreatePool(vout_display_t *vd, video_format_t *fmt)
810 {
811     vout_display_sys_t *sys = vd->sys;
812     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
813
814     /* */
815     *fmt = vd->source;
816
817     /* Find the appropriate D3DFORMAT for the render chroma, the format will be the closest to
818      * the requested chroma which is usable by the hardware in an offscreen surface, as they
819      * typically support more formats than textures */
820     const d3d_format_t *d3dfmt = Direct3DFindFormat(vd, fmt->i_chroma, sys->d3dpp.BackBufferFormat);
821     if (!d3dfmt) {
822         msg_Err(vd, "surface pixel format is not supported.");
823         return VLC_EGENERIC;
824     }
825     fmt->i_chroma = d3dfmt->fourcc;
826     fmt->i_rmask  = d3dfmt->rmask;
827     fmt->i_gmask  = d3dfmt->gmask;
828     fmt->i_bmask  = d3dfmt->bmask;
829
830     /* We create one picture.
831      * It is useless to create more as we can't be used for direct rendering */
832
833     /* Create a surface */
834     LPDIRECT3DSURFACE9 surface;
835     HRESULT hr = IDirect3DDevice9_CreateOffscreenPlainSurface(d3ddev,
836                                                               fmt->i_width,
837                                                               fmt->i_height,
838                                                               d3dfmt->format,
839                                                               D3DPOOL_DEFAULT,
840                                                               &surface,
841                                                               NULL);
842     if (FAILED(hr)) {
843         msg_Err(vd, "Failed to create picture surface. (hr=0x%lx)", hr);
844         return VLC_EGENERIC;
845     }
846     /* fill surface with black color */
847     IDirect3DDevice9_ColorFill(d3ddev, surface, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0));
848
849     /* Create the associated picture */
850     picture_resource_t *rsc = &sys->resource;
851     rsc->p_sys = malloc(sizeof(*rsc->p_sys));
852     if (!rsc->p_sys) {
853         IDirect3DSurface9_Release(surface);
854         return VLC_ENOMEM;
855     }
856     rsc->p_sys->surface = surface;
857     for (int i = 0; i < PICTURE_PLANE_MAX; i++) {
858         rsc->p[i].p_pixels = NULL;
859         rsc->p[i].i_pitch = 0;
860         rsc->p[i].i_lines = fmt->i_height / (i > 0 ? 2 : 1);
861     }
862     picture_t *picture = picture_NewFromResource(fmt, rsc);
863     if (!picture) {
864         IDirect3DSurface9_Release(surface);
865         free(rsc->p_sys);
866         return VLC_ENOMEM;
867     }
868
869     /* Wrap it into a picture pool */
870     picture_pool_configuration_t pool_cfg;
871     memset(&pool_cfg, 0, sizeof(pool_cfg));
872     pool_cfg.picture_count = 1;
873     pool_cfg.picture       = &picture;
874     pool_cfg.lock          = Direct3DLockSurface;
875     pool_cfg.unlock        = Direct3DUnlockSurface;
876
877     sys->pool = picture_pool_NewExtended(&pool_cfg);
878     if (!sys->pool) {
879         picture_Release(picture);
880         IDirect3DSurface9_Release(surface);
881         return VLC_ENOMEM;
882     }
883     return VLC_SUCCESS;
884 }
885 /**
886  * It destroys the pool of picture and its resources.
887  */
888 static void Direct3DDestroyPool(vout_display_t *vd)
889 {
890     vout_display_sys_t *sys = vd->sys;
891
892     if (sys->pool) {
893         picture_resource_t *rsc = &sys->resource;
894         IDirect3DSurface9_Release(rsc->p_sys->surface);
895
896         picture_pool_Delete(sys->pool);
897     }
898     sys->pool = NULL;
899 }
900
901 /* */
902 typedef struct
903 {
904     FLOAT       x,y,z;      // vertex untransformed position
905     FLOAT       rhw;        // eye distance
906     D3DCOLOR    diffuse;    // diffuse color
907     FLOAT       tu, tv;     // texture relative coordinates
908 } CUSTOMVERTEX;
909
910 #define D3DFVF_CUSTOMVERTEX (D3DFVF_XYZRHW|D3DFVF_DIFFUSE|D3DFVF_TEX1)
911
912 /**
913  * It allocates and initializes the resources needed to render the scene.
914  */
915 static int Direct3DCreateScene(vout_display_t *vd)
916 {
917     vout_display_sys_t *sys = vd->sys;
918     LPDIRECT3DDEVICE9       d3ddev = sys->d3ddev;
919     HRESULT hr;
920
921     /*
922      * Create a texture for use when rendering a scene
923      * for performance reason, texture format is identical to backbuffer
924      * which would usually be a RGB format
925      */
926     LPDIRECT3DTEXTURE9 d3dtex;
927     hr = IDirect3DDevice9_CreateTexture(d3ddev,
928                                         sys->d3dpp.BackBufferWidth,
929                                         sys->d3dpp.BackBufferHeight,
930                                         1,
931                                         D3DUSAGE_RENDERTARGET,
932                                         sys->d3dpp.BackBufferFormat,
933                                         D3DPOOL_DEFAULT,
934                                         &d3dtex,
935                                         NULL);
936     if (FAILED(hr)) {
937         msg_Err(vd, "Failed to create texture. (hr=0x%lx)", hr);
938         return VLC_EGENERIC;
939     }
940
941     /*
942     ** Create a vertex buffer for use when rendering scene
943     */
944     LPDIRECT3DVERTEXBUFFER9 d3dvtc;
945     hr = IDirect3DDevice9_CreateVertexBuffer(d3ddev,
946                                              sizeof(CUSTOMVERTEX)*4,
947                                              D3DUSAGE_DYNAMIC|D3DUSAGE_WRITEONLY,
948                                              D3DFVF_CUSTOMVERTEX,
949                                              D3DPOOL_DEFAULT,
950                                              &d3dvtc,
951                                              NULL);
952     if (FAILED(hr)) {
953         msg_Err(vd, "Failed to create vertex buffer. (hr=0x%lx)", hr);
954         IDirect3DTexture9_Release(d3dtex);
955         return VLC_EGENERIC;
956     }
957
958     /* */
959     sys->d3dtex = d3dtex;
960     sys->d3dvtc = d3dvtc;
961
962     // Texture coordinates outside the range [0.0, 1.0] are set
963     // to the texture color at 0.0 or 1.0, respectively.
964     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_ADDRESSU, D3DTADDRESS_CLAMP);
965     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_ADDRESSV, D3DTADDRESS_CLAMP);
966
967     // Set linear filtering quality
968     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR);
969     IDirect3DDevice9_SetSamplerState(d3ddev, 0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR);
970
971     // set maximum ambient light
972     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_AMBIENT, D3DCOLOR_XRGB(255,255,255));
973
974     // Turn off culling
975     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_CULLMODE, D3DCULL_NONE);
976
977     // Turn off the zbuffer
978     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ZENABLE, D3DZB_FALSE);
979
980     // Turn off lights
981     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_LIGHTING, FALSE);
982
983     // Enable dithering
984     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_DITHERENABLE, TRUE);
985
986     // disable stencil
987     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_STENCILENABLE, FALSE);
988
989     // manage blending
990     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHABLENDENABLE, TRUE);
991     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_SRCBLEND,D3DBLEND_SRCALPHA);
992     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_DESTBLEND,D3DBLEND_INVSRCALPHA);
993     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHATESTENABLE,TRUE);
994     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHAREF, 0x10);
995     IDirect3DDevice9_SetRenderState(d3ddev, D3DRS_ALPHAFUNC,D3DCMP_GREATER);
996
997     // Set texture states
998     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_COLOROP,D3DTOP_MODULATE);
999     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_COLORARG1,D3DTA_TEXTURE);
1000     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_COLORARG2,D3DTA_DIFFUSE);
1001
1002     // turn off alpha operation
1003     IDirect3DDevice9_SetTextureStageState(d3ddev, 0, D3DTSS_ALPHAOP, D3DTOP_DISABLE);
1004
1005     msg_Dbg(vd, "Direct3D scene created successfully");
1006
1007     return VLC_SUCCESS;
1008 }
1009
1010 /**
1011  * It releases the scene resources.
1012  */
1013 static void Direct3DDestroyScene(vout_display_t *vd)
1014 {
1015     vout_display_sys_t *sys = vd->sys;
1016
1017     LPDIRECT3DVERTEXBUFFER9 d3dvtc = sys->d3dvtc;
1018     if (d3dvtc)
1019         IDirect3DVertexBuffer9_Release(d3dvtc);
1020
1021     LPDIRECT3DTEXTURE9 d3dtex = sys->d3dtex;
1022     if (d3dtex)
1023         IDirect3DTexture9_Release(d3dtex);
1024
1025     sys->d3dvtc = NULL;
1026     sys->d3dtex = NULL;
1027     msg_Dbg(vd, "Direct3D scene released successfully");
1028 }
1029
1030 /**
1031  * It copies picture surface into a texture and renders into a scene.
1032  *
1033  * This function is intented for higher end 3D cards, with pixel shader support
1034  * and at least 64 MiB of video RAM.
1035  */
1036 static void Direct3DRenderScene(vout_display_t *vd, LPDIRECT3DSURFACE9 surface)
1037 {
1038     vout_display_sys_t *sys = vd->sys;
1039     LPDIRECT3DDEVICE9 d3ddev = sys->d3ddev;
1040     HRESULT hr;
1041
1042     // check if device is still available
1043     hr = IDirect3DDevice9_TestCooperativeLevel(d3ddev);
1044     if (FAILED(hr)) {
1045         if (hr == D3DERR_DEVICENOTRESET && !sys->reset_device) {
1046             vout_display_SendEventPicturesInvalid(vd);
1047             sys->reset_device = true;
1048         }
1049         return;
1050     }
1051     /* */
1052     LPDIRECT3DTEXTURE9      d3dtex  = sys->d3dtex;
1053     LPDIRECT3DVERTEXBUFFER9 d3dvtc  = sys->d3dvtc;
1054
1055     /* Clear the backbuffer and the zbuffer */
1056     hr = IDirect3DDevice9_Clear(d3ddev, 0, NULL, D3DCLEAR_TARGET,
1057                               D3DCOLOR_XRGB(0, 0, 0), 1.0f, 0);
1058     if (FAILED(hr)) {
1059         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1060         return;
1061     }
1062     /*  retrieve picture surface */
1063     LPDIRECT3DSURFACE9 d3dsrc = surface;
1064     if (!d3dsrc) {
1065         msg_Dbg(vd, "no surface to render ?");
1066         return;
1067     }
1068
1069     /* retrieve texture top-level surface */
1070     LPDIRECT3DSURFACE9 d3ddest;
1071     hr = IDirect3DTexture9_GetSurfaceLevel(d3dtex, 0, &d3ddest);
1072     if (FAILED(hr)) {
1073         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1074         return;
1075     }
1076
1077     /* Copy picture surface into texture surface
1078      * color space conversion and scaling happen here */
1079     RECT src = vd->sys->rect_src_clipped;
1080     RECT dst = vd->sys->rect_dest_clipped;
1081
1082     hr = IDirect3DDevice9_StretchRect(d3ddev, d3dsrc, &src, d3ddest, &dst, D3DTEXF_LINEAR);
1083     IDirect3DSurface9_Release(d3ddest);
1084     if (FAILED(hr)) {
1085         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1086         return;
1087     }
1088
1089     /* Update the vertex buffer */
1090     CUSTOMVERTEX *vertices;
1091     hr = IDirect3DVertexBuffer9_Lock(d3dvtc, 0, 0, &vertices, D3DLOCK_DISCARD);
1092     if (FAILED(hr)) {
1093         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1094         return;
1095     }
1096
1097     /* Setup vertices */
1098     const float f_width  = vd->sys->d3dpp.BackBufferWidth;
1099     const float f_height = vd->sys->d3dpp.BackBufferHeight;
1100
1101     /* -0.5f is a "feature" of DirectX and it seems to apply to Direct3d also */
1102     /* http://www.sjbrown.co.uk/2003/05/01/fix-directx-rasterisation/ */
1103     vertices[0].x       = -0.5f;       // left
1104     vertices[0].y       = -0.5f;       // top
1105     vertices[0].z       = 0.0f;
1106     vertices[0].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1107     vertices[0].rhw     = 1.0f;
1108     vertices[0].tu      = 0.0f;
1109     vertices[0].tv      = 0.0f;
1110
1111     vertices[1].x       = f_width - 0.5f;    // right
1112     vertices[1].y       = -0.5f;       // top
1113     vertices[1].z       = 0.0f;
1114     vertices[1].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1115     vertices[1].rhw     = 1.0f;
1116     vertices[1].tu      = 1.0f;
1117     vertices[1].tv      = 0.0f;
1118
1119     vertices[2].x       = f_width - 0.5f;    // right
1120     vertices[2].y       = f_height - 0.5f;   // bottom
1121     vertices[2].z       = 0.0f;
1122     vertices[2].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1123     vertices[2].rhw     = 1.0f;
1124     vertices[2].tu      = 1.0f;
1125     vertices[2].tv      = 1.0f;
1126
1127     vertices[3].x       = -0.5f;       // left
1128     vertices[3].y       = f_height - 0.5f;   // bottom
1129     vertices[3].z       = 0.0f;
1130     vertices[3].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1131     vertices[3].rhw     = 1.0f;
1132     vertices[3].tu      = 0.0f;
1133     vertices[3].tv      = 1.0f;
1134
1135     hr= IDirect3DVertexBuffer9_Unlock(d3dvtc);
1136     if (FAILED(hr)) {
1137         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1138         return;
1139     }
1140
1141     // Begin the scene
1142     hr = IDirect3DDevice9_BeginScene(d3ddev);
1143     if (FAILED(hr)) {
1144         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1145         return;
1146     }
1147
1148     // Setup our texture. Using textures introduces the texture stage states,
1149     // which govern how textures get blended together (in the case of multiple
1150     // textures) and lighting information. In this case, we are modulating
1151     // (blending) our texture with the diffuse color of the vertices.
1152     hr = IDirect3DDevice9_SetTexture(d3ddev, 0, (LPDIRECT3DBASETEXTURE9)d3dtex);
1153     if (FAILED(hr)) {
1154         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1155         IDirect3DDevice9_EndScene(d3ddev);
1156         return;
1157     }
1158
1159     // Render the vertex buffer contents
1160     hr = IDirect3DDevice9_SetStreamSource(d3ddev, 0, d3dvtc, 0, sizeof(CUSTOMVERTEX));
1161     if (FAILED(hr)) {
1162         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1163         IDirect3DDevice9_EndScene(d3ddev);
1164         return;
1165     }
1166
1167     // we use FVF instead of vertex shader
1168     hr = IDirect3DDevice9_SetFVF(d3ddev, D3DFVF_CUSTOMVERTEX);
1169     if (FAILED(hr)) {
1170         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1171         IDirect3DDevice9_EndScene(d3ddev);
1172         return;
1173     }
1174
1175     // draw rectangle
1176     hr = IDirect3DDevice9_DrawPrimitive(d3ddev, D3DPT_TRIANGLEFAN, 0, 2);
1177     if (FAILED(hr)) {
1178         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1179         IDirect3DDevice9_EndScene(d3ddev);
1180         return;
1181     }
1182
1183     // End the scene
1184     hr = IDirect3DDevice9_EndScene(d3ddev);
1185     if (FAILED(hr)) {
1186         msg_Dbg(vd, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1187         return;
1188     }
1189 }
1190
1191 /*****************************************************************************
1192  * DesktopCallback: desktop mode variable callback
1193  *****************************************************************************/
1194 static int DesktopCallback(vlc_object_t *object, char const *psz_cmd,
1195                             vlc_value_t oldval, vlc_value_t newval,
1196                             void *p_data)
1197 {
1198     vout_display_t *vd = (vout_display_t *)object;
1199     vout_display_sys_t *sys = vd->sys;
1200     VLC_UNUSED(psz_cmd);
1201     VLC_UNUSED(oldval);
1202     VLC_UNUSED(p_data);
1203
1204     vlc_mutex_lock(&sys->lock);
1205     const bool ch_desktop = !sys->desktop_requested != !newval.b_bool;
1206     sys->ch_desktop |= ch_desktop;
1207     sys->desktop_requested = newval.b_bool;
1208     vlc_mutex_unlock(&sys->lock);
1209
1210     /* FIXME we should have a way to export variable to be saved */
1211     if (ch_desktop) {
1212         playlist_t *p_playlist = pl_Get(vd);
1213         /* Modify playlist as well because the vout might have to be
1214          * restarted */
1215         var_Create(p_playlist, "direct3d-desktop", VLC_VAR_BOOL);
1216         var_SetBool(p_playlist, "direct3d-desktop", newval.b_bool);
1217     }
1218     return VLC_SUCCESS;
1219 }