Add a desktop mode to the Direct3d video output. This allow displaying video on a...
[vlc.git] / modules / video_output / msw / direct3d.c
1 /*****************************************************************************
2  * direct3d.c: Windows Direct3D video output module
3  *****************************************************************************
4  * Copyright (C) 2006-2009 the VideoLAN team
5  *$Id$
6  *
7  * Authors: Damien Fouilleul <damienf@videolan.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 /*****************************************************************************
25  * Preamble:
26  *
27  * This plugin will use YUV surface if supported, using YUV will result in
28  * the best video quality (hardware filering when rescaling the picture)
29  * and the fastest display as it requires less processing.
30  *
31  * If YUV overlay is not supported this plugin will use RGB offscreen video
32  * surfaces that will be blitted onto the primary surface (display) to
33  * effectively display the pictures.
34  *
35  *****************************************************************************/
36 #include <errno.h>                                                 /* ENOMEM */
37
38 #ifdef HAVE_CONFIG_H
39 # include "config.h"
40 #endif
41
42 #include <vlc_common.h>
43 #include <vlc_plugin.h>
44 #include <vlc_interface.h>
45 #include <vlc_playlist.h>
46 #include <vlc_vout.h>
47
48 #include <windows.h>
49 #include <d3d9.h>
50
51 #include "vout.h"
52
53 /*****************************************************************************
54  * Local prototypes.
55  *****************************************************************************/
56 static int  OpenVideo  ( vlc_object_t * );
57 static void CloseVideo ( vlc_object_t * );
58
59 static int  Init      ( vout_thread_t * );
60 static void End       ( vout_thread_t * );
61 static int  Manage    ( vout_thread_t * );
62 static void Display   ( vout_thread_t *, picture_t * );
63 static void FirstDisplay( vout_thread_t *, picture_t * );
64
65 static int Direct3DVoutCreate     ( vout_thread_t * );
66 static void Direct3DVoutRelease   ( vout_thread_t * );
67
68 static int  Direct3DVoutOpen      ( vout_thread_t * );
69 static void Direct3DVoutClose     ( vout_thread_t * );
70
71 static int Direct3DVoutResetDevice( vout_thread_t * );
72
73 static int Direct3DVoutCreatePictures   ( vout_thread_t *, size_t );
74 static void Direct3DVoutReleasePictures ( vout_thread_t * );
75
76 static int Direct3DVoutLockSurface  ( vout_thread_t *, picture_t * );
77 static int Direct3DVoutUnlockSurface( vout_thread_t *, picture_t * );
78
79 static int Direct3DVoutCreateScene      ( vout_thread_t * );
80 static void Direct3DVoutReleaseScene    ( vout_thread_t * );
81 static void Direct3DVoutRenderScene     ( vout_thread_t *, picture_t * );
82
83 static int DesktopCallback( vlc_object_t *p_this, char const *psz_cmd,
84                             vlc_value_t oldval, vlc_value_t newval,
85                             void *p_data );
86
87 /*****************************************************************************
88  * Module descriptor
89  *****************************************************************************/
90
91 static bool IsVistaOrAbove(void)
92 {
93     OSVERSIONINFO winVer;
94     winVer.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
95
96     if( GetVersionEx(&winVer) )
97     {
98         if( winVer.dwMajorVersion > 5 )
99         {
100             /* Windows Vista or above, make this module the default */
101             return true;
102         }
103     }
104     /* Windows XP or lower, make sure this module isn't the default */
105     return false;
106 }
107
108 static int OpenVideoXP( vlc_object_t *obj )
109 {
110     return IsVistaOrAbove() ? VLC_EGENERIC : OpenVideo( obj );
111 }
112
113 static int OpenVideoVista( vlc_object_t *obj )
114 {
115     return IsVistaOrAbove() ? OpenVideo( obj ) : VLC_EGENERIC;
116 }
117
118 #define DESKTOP_TEXT N_("Enable desktop mode ")
119 #define DESKTOP_LONGTEXT N_( \
120     "The desktop mode allows you to display the video on the desktop." )
121
122 vlc_module_begin ()
123     set_shortname( "Direct3D" )
124     set_category( CAT_VIDEO )
125     set_subcategory( SUBCAT_VIDEO_VOUT )
126
127     add_bool( "direct3d-desktop", 0, NULL, DESKTOP_TEXT, DESKTOP_LONGTEXT,
128               true )
129
130     set_description( N_("DirectX 3D video output") )
131     set_capability( "video output", 50 )
132     add_shortcut( "direct3d" )
133     set_callbacks( OpenVideoXP, CloseVideo )
134
135     /* FIXME: Hack to avoid unregistering our window class */
136     linked_with_a_crap_library_which_uses_atexit ()
137
138     add_submodule()
139         set_capability( "video output", 150 )
140         add_shortcut( "direct3d" )
141         set_callbacks( OpenVideoVista, CloseVideo )
142 vlc_module_end ()
143
144 #if 0 /* FIXME */
145     /* check if we registered a window class because we need to
146      * unregister it */
147     WNDCLASS wndclass;
148     if( GetClassInfo( GetModuleHandle(NULL), "VLC DirectX", &wndclass ) )
149         UnregisterClass( "VLC DirectX", GetModuleHandle(NULL) );
150 #endif
151
152 /*****************************************************************************
153  * CUSTOMVERTEX:
154  *****************************************************************************
155  *****************************************************************************/
156 typedef struct
157 {
158     FLOAT       x,y,z;      // vertex untransformed position
159     FLOAT       rhw;        // eye distance
160     D3DCOLOR    diffuse;    // diffuse color
161     FLOAT       tu, tv;     // texture relative coordinates
162 } CUSTOMVERTEX;
163
164 #define D3DFVF_CUSTOMVERTEX (D3DFVF_XYZRHW|D3DFVF_DIFFUSE|D3DFVF_TEX1)
165
166 /*****************************************************************************
167  * OpenVideo: allocate Vout video thread output method
168  *****************************************************************************
169  * This function allocates and initialize the Direct3D vout method.
170  *****************************************************************************/
171 static int OpenVideo( vlc_object_t *p_this )
172 {
173     vlc_value_t val;
174     vout_thread_t * p_vout = (vout_thread_t *)p_this;
175
176     /* Allocate structure */
177     p_vout->p_sys = calloc( 1, sizeof( vout_sys_t ) );
178     if( p_vout->p_sys == NULL )
179         return VLC_ENOMEM;
180
181     if( VLC_SUCCESS != Direct3DVoutCreate( p_vout ) )
182     {
183         msg_Err( p_vout, "Direct3D could not be initialized !");
184         Direct3DVoutRelease( p_vout );
185         free( p_vout->p_sys );
186         return VLC_EGENERIC;
187     }
188
189     /* Initialisations */
190     p_vout->pf_init = Init;
191     p_vout->pf_end = End;
192     p_vout->pf_manage = Manage;
193     p_vout->pf_render = Direct3DVoutRenderScene;
194     p_vout->pf_display = FirstDisplay;
195
196     p_vout->p_sys->hwnd = p_vout->p_sys->hvideownd = NULL;
197     p_vout->p_sys->hparent = p_vout->p_sys->hfswnd = NULL;
198     p_vout->p_sys->i_changes = 0;
199     p_vout->p_sys->b_desktop = false;
200     vlc_mutex_init( &p_vout->p_sys->lock );
201     SetRectEmpty( &p_vout->p_sys->rect_display );
202     SetRectEmpty( &p_vout->p_sys->rect_parent );
203
204     var_Create( p_vout, "directx-hw-yuv", VLC_VAR_BOOL | VLC_VAR_DOINHERIT );
205     var_Create( p_vout, "directx-device", VLC_VAR_STRING | VLC_VAR_DOINHERIT );
206
207     p_vout->p_sys->b_cursor_hidden = 0;
208     p_vout->p_sys->i_lastmoved = mdate();
209     p_vout->p_sys->i_mouse_hide_timeout =
210         var_GetInteger(p_vout, "mouse-hide-timeout") * 1000;
211
212     var_Create( p_vout, "video-title", VLC_VAR_STRING | VLC_VAR_DOINHERIT );
213     var_Create( p_vout, "disable-screensaver", VLC_VAR_BOOL | VLC_VAR_DOINHERIT );
214
215     /* Set main window's size */
216     p_vout->p_sys->i_window_width = p_vout->i_window_width;
217     p_vout->p_sys->i_window_height = p_vout->i_window_height;
218
219     if ( CreateEventThread( p_vout ) )
220     {
221         /* Variable to indicate if the window should be on top of others */
222         /* Trigger a callback right now */
223         var_TriggerCallback( p_vout, "video-on-top" );
224
225         /* Trigger a callback right now */
226         var_Create( p_vout, "direct3d-desktop", VLC_VAR_BOOL|VLC_VAR_DOINHERIT );
227         val.psz_string = _("Desktop");
228         var_Change( p_vout, "direct3d-desktop", VLC_VAR_SETTEXT, &val, NULL );
229         var_AddCallback( p_vout, "direct3d-desktop", DesktopCallback, NULL );
230         var_TriggerCallback( p_vout, "direct3d-desktop" );
231
232         DisableScreensaver ( p_vout );
233
234         return VLC_SUCCESS;
235     }
236     else
237     {
238         CloseVideo( VLC_OBJECT(p_vout) );
239         return VLC_EGENERIC;
240     }
241 }
242
243 /*****************************************************************************
244  * CloseVideo: destroy Sys video thread output method
245  *****************************************************************************
246  * Terminate an output method created by Create
247  *****************************************************************************/
248 static void CloseVideo( vlc_object_t *p_this )
249 {
250     vout_thread_t * p_vout = (vout_thread_t *)p_this;
251
252     Direct3DVoutRelease( p_vout );
253
254     StopEventThread( p_vout );
255
256     RestoreScreensaver( p_vout );
257
258     free( p_vout->p_sys );
259     p_vout->p_sys = NULL;
260 }
261
262 /*****************************************************************************
263  * Init: initialize Direct3D video thread output method
264  *****************************************************************************/
265 static int Init( vout_thread_t *p_vout )
266 {
267     int i_ret;
268
269     p_vout->p_sys->b_hw_yuv = var_GetBool( p_vout, "directx-hw-yuv" );
270
271     /* Initialise Direct3D */
272     if( VLC_SUCCESS != Direct3DVoutOpen( p_vout ) )
273     {
274         msg_Err( p_vout, "cannot initialize Direct3D" );
275         return VLC_EGENERIC;
276     }
277
278     /* Initialize the output structure.
279      * Since Direct3D can do rescaling for us, stick to the default
280      * coordinates and aspect. */
281     p_vout->output.i_width  = p_vout->render.i_width;
282     p_vout->output.i_height = p_vout->render.i_height;
283     p_vout->output.i_aspect = p_vout->render.i_aspect;
284     p_vout->fmt_out = p_vout->fmt_in;
285     UpdateRects( p_vout, true );
286
287     /*  create picture pool */
288     p_vout->output.i_chroma = 0;
289     i_ret = Direct3DVoutCreatePictures(p_vout, 1);
290     if( VLC_SUCCESS != i_ret )
291     {
292         msg_Err(p_vout, "Direct3D picture pool initialization failed !");
293         return i_ret;
294     }
295
296     /* create scene */
297     i_ret = Direct3DVoutCreateScene(p_vout);
298     if( VLC_SUCCESS != i_ret )
299     {
300         msg_Err(p_vout, "Direct3D scene initialization failed !");
301         Direct3DVoutReleasePictures(p_vout);
302         return i_ret;
303     }
304
305     /* Change the window title bar text */
306     PostMessage( p_vout->p_sys->hwnd, WM_VLC_CHANGE_TEXT, 0, 0 );
307
308     p_vout->fmt_out.i_chroma = p_vout->output.i_chroma;
309     return VLC_SUCCESS;
310 }
311
312 /*****************************************************************************
313  * End: terminate Sys video thread output method
314  *****************************************************************************
315  * Terminate an output method created by Create.
316  * It is called at the end of the thread.
317  *****************************************************************************/
318 static void End( vout_thread_t *p_vout )
319 {
320     Direct3DVoutReleaseScene(p_vout);
321     Direct3DVoutReleasePictures(p_vout);
322     Direct3DVoutClose( p_vout );
323 }
324
325 /*****************************************************************************
326  * Manage: handle Sys events
327  *****************************************************************************
328  * This function should be called regularly by the video output thread.
329  * It returns a non null value if an error occurred.
330  *****************************************************************************/
331 static int Manage( vout_thread_t *p_vout )
332 {
333     /* If we do not control our window, we check for geometry changes
334      * ourselves because the parent might not send us its events. */
335     vlc_mutex_lock( &p_vout->p_sys->lock );
336     if( p_vout->p_sys->hparent && !p_vout->b_fullscreen )
337     {
338         RECT rect_parent;
339         POINT point;
340
341         vlc_mutex_unlock( &p_vout->p_sys->lock );
342
343         GetClientRect( p_vout->p_sys->hparent, &rect_parent );
344         point.x = point.y = 0;
345         ClientToScreen( p_vout->p_sys->hparent, &point );
346         OffsetRect( &rect_parent, point.x, point.y );
347
348         if( !EqualRect( &rect_parent, &p_vout->p_sys->rect_parent ) )
349         {
350             p_vout->p_sys->rect_parent = rect_parent;
351
352             SetWindowPos( p_vout->p_sys->hwnd, 0, 0, 0,
353                           rect_parent.right - rect_parent.left,
354                           rect_parent.bottom - rect_parent.top,
355                           SWP_NOZORDER );
356             UpdateRects( p_vout, true );
357         }
358     }
359     else
360     {
361         vlc_mutex_unlock( &p_vout->p_sys->lock );
362     }
363
364     /*
365      * Position Change
366      */
367     if( p_vout->p_sys->i_changes & DX_POSITION_CHANGE )
368     {
369 #if 0 /* need that when bicubic filter is available */
370         RECT rect;
371         UINT width, height;
372
373         GetClientRect(p_vout->p_sys->hvideownd, &rect);
374         width  = rect.right-rect.left;
375         height = rect.bottom-rect.top;
376
377         if( (width != p_vout->p_sys->d3dpp.BackBufferWidth)
378          || (height != p_vout->p_sys->d3dpp.BackBufferHeight) )
379         {
380             msg_Dbg(p_vout, "resizing device back buffers to (%lux%lu)", width, height);
381             // need to reset D3D device to resize back buffer
382             if( VLC_SUCCESS != Direct3DVoutResetDevice(p_vout, width, height) )
383                 return VLC_EGENERIC;
384         }
385 #endif
386         p_vout->p_sys->i_changes &= ~DX_POSITION_CHANGE;
387     }
388     
389     /*
390             * Desktop mode change
391             */
392     if( p_vout->p_sys->i_changes & DX_DESKTOP_CHANGE )
393     {
394         /* Close the direct3d instance attached to the current output window. */
395         End( p_vout );
396         StopEventThread( p_vout );
397         /* Set the switching mode flag */
398         p_vout->p_sys->i_changes |= SWITCHING_MODE_FLAG;
399         /* Reset the flag */
400         p_vout->p_sys->i_changes &= ~DX_DESKTOP_CHANGE;
401     }
402     
403     if( p_vout->p_sys->i_changes & EVENT_THREAD_ENDED
404         && p_vout->p_sys->i_changes & SWITCHING_MODE_FLAG )
405     {
406         /* Open the direct3d output and attaches it to the new window */
407         p_vout->p_sys->b_desktop = !p_vout->p_sys->b_desktop;
408         p_vout->pf_display = FirstDisplay;
409
410         CreateEventThread( p_vout );
411         Init( p_vout );
412
413         /* Reset the flags */
414         p_vout->p_sys->i_changes &= ~EVENT_THREAD_ENDED;
415         p_vout->p_sys->i_changes &= ~SWITCHING_MODE_FLAG;
416     }
417
418     /* autoscale toggle */
419     if( p_vout->i_changes & VOUT_SCALE_CHANGE )
420     {
421         p_vout->i_changes &= ~VOUT_SCALE_CHANGE;
422
423         p_vout->b_autoscale = var_GetBool( p_vout, "autoscale" );
424         p_vout->i_zoom = (int) ZOOM_FP_FACTOR;
425
426         UpdateRects( p_vout, true );
427     }
428
429     /* scaling factor */
430     if( p_vout->i_changes & VOUT_ZOOM_CHANGE )
431     {
432         p_vout->i_changes &= ~VOUT_ZOOM_CHANGE;
433
434         p_vout->b_autoscale = false;
435         p_vout->i_zoom =
436             (int)( ZOOM_FP_FACTOR * var_GetFloat( p_vout, "scale" ) );
437         UpdateRects( p_vout, true );
438     }
439
440     /* Check for cropping / aspect changes */
441     if( p_vout->i_changes & VOUT_CROP_CHANGE ||
442         p_vout->i_changes & VOUT_ASPECT_CHANGE )
443     {
444         p_vout->i_changes &= ~VOUT_CROP_CHANGE;
445         p_vout->i_changes &= ~VOUT_ASPECT_CHANGE;
446
447         p_vout->fmt_out.i_x_offset = p_vout->fmt_in.i_x_offset;
448         p_vout->fmt_out.i_y_offset = p_vout->fmt_in.i_y_offset;
449         p_vout->fmt_out.i_visible_width = p_vout->fmt_in.i_visible_width;
450         p_vout->fmt_out.i_visible_height = p_vout->fmt_in.i_visible_height;
451         p_vout->fmt_out.i_aspect = p_vout->fmt_in.i_aspect;
452         p_vout->fmt_out.i_sar_num = p_vout->fmt_in.i_sar_num;
453         p_vout->fmt_out.i_sar_den = p_vout->fmt_in.i_sar_den;
454         p_vout->output.i_aspect = p_vout->fmt_in.i_aspect;
455         UpdateRects( p_vout, true );
456     }
457
458     /* We used to call the Win32 PeekMessage function here to read the window
459      * messages. But since window can stay blocked into this function for a
460      * long time (for example when you move your window on the screen), I
461      * decided to isolate PeekMessage in another thread. */
462
463     /*
464      * Fullscreen change
465      */
466     if( p_vout->i_changes & VOUT_FULLSCREEN_CHANGE
467         || p_vout->p_sys->i_changes & VOUT_FULLSCREEN_CHANGE )
468     {
469         Win32ToggleFullscreen( p_vout );
470
471         p_vout->i_changes &= ~VOUT_FULLSCREEN_CHANGE;
472         p_vout->p_sys->i_changes &= ~VOUT_FULLSCREEN_CHANGE;
473     }
474
475     /*
476      * Pointer change
477      */
478     if( p_vout->b_fullscreen && !p_vout->p_sys->b_cursor_hidden &&
479         (mdate() - p_vout->p_sys->i_lastmoved) >
480             p_vout->p_sys->i_mouse_hide_timeout )
481     {
482         POINT point;
483         HWND hwnd;
484
485         /* Hide the cursor only if it is inside our window */
486         GetCursorPos( &point );
487         hwnd = WindowFromPoint(point);
488         if( hwnd == p_vout->p_sys->hwnd || hwnd == p_vout->p_sys->hvideownd )
489         {
490             PostMessage( p_vout->p_sys->hwnd, WM_VLC_HIDE_MOUSE, 0, 0 );
491         }
492         else
493         {
494             p_vout->p_sys->i_lastmoved = mdate();
495         }
496     }
497
498     /*
499      * "Always on top" status change
500      */
501     if( p_vout->p_sys->b_on_top_change )
502     {
503         HMENU hMenu = GetSystemMenu( p_vout->p_sys->hwnd, FALSE );
504         bool b = var_GetBool( p_vout, "video-on-top" );
505
506         /* Set the window on top if necessary */
507         if( b && !( GetWindowLong( p_vout->p_sys->hwnd, GWL_EXSTYLE )
508                            & WS_EX_TOPMOST ) )
509         {
510             CheckMenuItem( hMenu, IDM_TOGGLE_ON_TOP,
511                            MF_BYCOMMAND | MFS_CHECKED );
512             SetWindowPos( p_vout->p_sys->hwnd, HWND_TOPMOST, 0, 0, 0, 0,
513                           SWP_NOSIZE | SWP_NOMOVE );
514         }
515         else
516         /* The window shouldn't be on top */
517         if( !b && ( GetWindowLong( p_vout->p_sys->hwnd, GWL_EXSTYLE )
518                            & WS_EX_TOPMOST ) )
519         {
520             CheckMenuItem( hMenu, IDM_TOGGLE_ON_TOP,
521                            MF_BYCOMMAND | MFS_UNCHECKED );
522             SetWindowPos( p_vout->p_sys->hwnd, HWND_NOTOPMOST, 0, 0, 0, 0,
523                           SWP_NOSIZE | SWP_NOMOVE );
524         }
525
526         p_vout->p_sys->b_on_top_change = false;
527     }
528
529     /* Check if the event thread is still running */
530     if( !vlc_object_alive (p_vout->p_sys->p_event) )
531     {
532         return VLC_EGENERIC; /* exit */
533     }
534
535     return VLC_SUCCESS;
536 }
537
538 /*****************************************************************************
539  * Display: displays previously rendered output
540  *****************************************************************************
541  * This function sends the currently rendered image to the display, wait until
542  * it is displayed and switch the two rendering buffers, preparing next frame.
543  *****************************************************************************/
544 static void Display( vout_thread_t *p_vout, picture_t *p_pic )
545 {
546     LPDIRECT3DDEVICE9       p_d3ddev = p_vout->p_sys->p_d3ddev;
547
548     if( p_vout->p_sys->i_changes & SWITCHING_MODE_FLAG )
549         return;
550
551     // Present the back buffer contents to the display
552     // stretching and filtering happens here
553     HRESULT hr = IDirect3DDevice9_Present(p_d3ddev,
554                     &(p_vout->p_sys->rect_src_clipped),
555                     NULL, NULL, NULL);
556     if( FAILED(hr) )
557         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
558 }
559
560 /*
561 ** this function is only used once when the first picture is received
562 ** this function will show the video window once a picture is ready
563 */
564
565 static void FirstDisplay( vout_thread_t *p_vout, picture_t *p_pic )
566 {
567     /* get initial picture presented through D3D */
568     Display(p_vout, p_pic);
569
570     /*
571     ** Video window is initially hidden, show it now since we got a
572     ** picture to show.
573     */
574     SetWindowPos( p_vout->p_sys->hvideownd, 0, 0, 0, 0, 0,
575         SWP_ASYNCWINDOWPOS|
576         SWP_FRAMECHANGED|
577         SWP_SHOWWINDOW|
578         SWP_NOMOVE|
579         SWP_NOSIZE|
580         SWP_NOZORDER );
581
582     /* use and restores proper display function for further pictures */
583     p_vout->pf_display = Display;
584 }
585
586 /*****************************************************************************
587  * DirectD3DVoutCreate: Initialize and instance of Direct3D9
588  *****************************************************************************
589  * This function initialize Direct3D and analyze available resources from
590  * default adapter.
591  *****************************************************************************/
592 static int Direct3DVoutCreate( vout_thread_t *p_vout )
593 {
594     HRESULT hr;
595     LPDIRECT3D9 p_d3dobj;
596     D3DCAPS9 d3dCaps;
597
598     LPDIRECT3D9 (WINAPI *OurDirect3DCreate9)(UINT SDKVersion);
599
600     p_vout->p_sys->hd3d9_dll = LoadLibrary(TEXT("D3D9.DLL"));
601     if( NULL == p_vout->p_sys->hd3d9_dll )
602     {
603         msg_Warn( p_vout, "cannot load d3d9.dll, aborting" );
604         return VLC_EGENERIC;
605     }
606
607     OurDirect3DCreate9 =
608       (void *)GetProcAddress( p_vout->p_sys->hd3d9_dll,
609                               TEXT("Direct3DCreate9") );
610     if( OurDirect3DCreate9 == NULL )
611     {
612         msg_Err( p_vout, "Cannot locate reference to Direct3DCreate9 ABI in DLL" );
613         return VLC_EGENERIC;
614     }
615
616     /* Create the D3D object. */
617     p_d3dobj = OurDirect3DCreate9( D3D_SDK_VERSION );
618     if( NULL == p_d3dobj )
619     {
620        msg_Err( p_vout, "Could not create Direct3D9 instance.");
621        return VLC_EGENERIC;
622     }
623     p_vout->p_sys->p_d3dobj = p_d3dobj;
624
625     /*
626     ** Get device capabilities
627     */
628     ZeroMemory(&d3dCaps, sizeof(d3dCaps));
629     hr = IDirect3D9_GetDeviceCaps(p_d3dobj, D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL, &d3dCaps);
630     if( FAILED(hr) )
631     {
632        msg_Err( p_vout, "Could not read adapter capabilities. (hr=0x%lX)", hr);
633        return VLC_EGENERIC;
634     }
635     /* TODO: need to test device capabilities and select the right render function */
636
637     return VLC_SUCCESS;
638 }
639
640 /*****************************************************************************
641  * DirectD3DVoutRelease: release an instance of Direct3D9
642  *****************************************************************************/
643
644 static void Direct3DVoutRelease( vout_thread_t *p_vout )
645 {
646     if( p_vout->p_sys->p_d3dobj )
647     {
648        IDirect3D9_Release(p_vout->p_sys->p_d3dobj);
649        p_vout->p_sys->p_d3dobj = NULL;
650     }
651     if( NULL != p_vout->p_sys->hd3d9_dll )
652     {
653         FreeLibrary(p_vout->p_sys->hd3d9_dll);
654         p_vout->p_sys->hd3d9_dll = NULL;
655     }
656 }
657
658 static int Direct3DFillPresentationParameters(vout_thread_t *p_vout, D3DPRESENT_PARAMETERS *d3dpp)
659 {
660     LPDIRECT3D9 p_d3dobj = p_vout->p_sys->p_d3dobj;
661     D3DDISPLAYMODE d3ddm;
662     HRESULT hr;
663
664     /*
665     ** Get the current desktop display mode, so we can set up a back
666     ** buffer of the same format
667     */
668     hr = IDirect3D9_GetAdapterDisplayMode(p_d3dobj, D3DADAPTER_DEFAULT, &d3ddm );
669     if( FAILED(hr))
670     {
671        msg_Err( p_vout, "Could not read adapter display mode. (hr=0x%lX)", hr);
672        return VLC_EGENERIC;
673     }
674
675     /* keep a copy of current desktop format */
676     p_vout->p_sys->bbFormat = d3ddm.Format;
677
678     /* Set up the structure used to create the D3DDevice. */
679     ZeroMemory( d3dpp, sizeof(D3DPRESENT_PARAMETERS) );
680     d3dpp->Flags                  = D3DPRESENTFLAG_VIDEO;
681     d3dpp->Windowed               = TRUE;
682     d3dpp->hDeviceWindow          = p_vout->p_sys->hvideownd;
683     d3dpp->BackBufferWidth        = p_vout->output.i_width;
684     d3dpp->BackBufferHeight       = p_vout->output.i_height;
685     d3dpp->SwapEffect             = D3DSWAPEFFECT_COPY;
686     d3dpp->MultiSampleType        = D3DMULTISAMPLE_NONE;
687     d3dpp->PresentationInterval   = D3DPRESENT_INTERVAL_DEFAULT;
688     d3dpp->BackBufferFormat       = d3ddm.Format;
689     d3dpp->BackBufferCount        = 1;
690     d3dpp->EnableAutoDepthStencil = FALSE;
691
692     return VLC_SUCCESS;
693 }
694
695 /*****************************************************************************
696  * DirectD3DVoutOpen: Takes care of all the Direct3D9 initialisations
697  *****************************************************************************
698  * This function creates Direct3D device
699  * this must be called from the vout thread for performance reason, as
700  * all Direct3D Device APIs are used in a non multithread safe environment
701  *****************************************************************************/
702 static int Direct3DVoutOpen( vout_thread_t *p_vout )
703 {
704     LPDIRECT3D9 p_d3dobj = p_vout->p_sys->p_d3dobj;
705     LPDIRECT3DDEVICE9 p_d3ddev;
706     D3DPRESENT_PARAMETERS d3dpp;
707     HRESULT hr;
708
709     if( VLC_SUCCESS != Direct3DFillPresentationParameters(p_vout, &d3dpp) )
710         return VLC_EGENERIC;
711
712     // Create the D3DDevice
713     hr = IDirect3D9_CreateDevice(p_d3dobj, D3DADAPTER_DEFAULT,
714                                  D3DDEVTYPE_HAL, p_vout->p_sys->hvideownd,
715                                  D3DCREATE_SOFTWARE_VERTEXPROCESSING|
716                                  D3DCREATE_MULTITHREADED,
717                                  &d3dpp, &p_d3ddev );
718     if( FAILED(hr) )
719     {
720        msg_Err(p_vout, "Could not create the D3D device! (hr=0x%lX)", hr);
721        return VLC_EGENERIC;
722     }
723     p_vout->p_sys->p_d3ddev = p_d3ddev;
724
725     msg_Dbg( p_vout, "Direct3D device adapter successfully initialized" );
726     return VLC_SUCCESS;
727 }
728
729 /*****************************************************************************
730  * DirectD3DClose: release the Direct3D9 device
731  *****************************************************************************/
732 static void Direct3DVoutClose( vout_thread_t *p_vout )
733 {
734     if( p_vout->p_sys->p_d3ddev )
735     {
736        IDirect3DDevice9_Release(p_vout->p_sys->p_d3ddev);
737        p_vout->p_sys->p_d3ddev = NULL;
738     }
739
740     p_vout->p_sys->hmonitor = NULL;
741 }
742
743 /*****************************************************************************
744  * DirectD3DClose: reset the Direct3D9 device
745  *****************************************************************************
746  * All resources must be deallocated before the reset occur, they will be
747  * realllocated once the reset has been performed successfully
748  *****************************************************************************/
749 static int Direct3DVoutResetDevice( vout_thread_t *p_vout )
750 {
751     LPDIRECT3DDEVICE9       p_d3ddev = p_vout->p_sys->p_d3ddev;
752     D3DPRESENT_PARAMETERS   d3dpp;
753     HRESULT hr;
754
755     if( VLC_SUCCESS != Direct3DFillPresentationParameters(p_vout, &d3dpp) )
756         return VLC_EGENERIC;
757
758     // release all D3D objects
759     Direct3DVoutReleaseScene( p_vout );
760     Direct3DVoutReleasePictures( p_vout );
761
762     hr = IDirect3DDevice9_Reset(p_d3ddev, &d3dpp);
763     if( SUCCEEDED(hr) )
764     {
765         // re-create them
766         if( (VLC_SUCCESS != Direct3DVoutCreatePictures(p_vout, 1))
767          || (VLC_SUCCESS != Direct3DVoutCreateScene(p_vout)) )
768         {
769             msg_Dbg(p_vout, "%s failed !", __FUNCTION__);
770             return VLC_EGENERIC;
771         }
772     }
773     else {
774         msg_Err(p_vout, "%s failed ! (hr=%08lX)", __FUNCTION__, hr);
775         return VLC_EGENERIC;
776     }
777     return VLC_SUCCESS;
778 }
779
780 static D3DFORMAT Direct3DVoutSelectFormat( vout_thread_t *p_vout, D3DFORMAT target,
781     const D3DFORMAT *formats, size_t count)
782 {
783     LPDIRECT3D9 p_d3dobj = p_vout->p_sys->p_d3dobj;
784     size_t c;
785
786     for( c=0; c<count; ++c )
787     {
788         HRESULT hr;
789         D3DFORMAT format = formats[c];
790         /* test whether device can create a surface of that format */
791         hr = IDirect3D9_CheckDeviceFormat(p_d3dobj, D3DADAPTER_DEFAULT,
792                 D3DDEVTYPE_HAL, target, 0, D3DRTYPE_SURFACE, format);
793         if( SUCCEEDED(hr) )
794         {
795             /* test whether device can perform color-conversion
796             ** from that format to target format
797             */
798             hr = IDirect3D9_CheckDeviceFormatConversion(p_d3dobj,
799                     D3DADAPTER_DEFAULT, D3DDEVTYPE_HAL,
800                     format, target);
801         }
802         if( SUCCEEDED(hr) )
803         {
804             // found a compatible format
805             switch( format )
806             {
807                 case D3DFMT_UYVY:
808                     msg_Dbg( p_vout, "selected surface pixel format is UYVY");
809                     break;
810                 case D3DFMT_YUY2:
811                     msg_Dbg( p_vout, "selected surface pixel format is YUY2");
812                     break;
813                 case D3DFMT_X8R8G8B8:
814                     msg_Dbg( p_vout, "selected surface pixel format is X8R8G8B8");
815                     break;
816                 case D3DFMT_A8R8G8B8:
817                     msg_Dbg( p_vout, "selected surface pixel format is A8R8G8B8");
818                     break;
819                 case D3DFMT_R8G8B8:
820                     msg_Dbg( p_vout, "selected surface pixel format is R8G8B8");
821                     break;
822                 case D3DFMT_R5G6B5:
823                     msg_Dbg( p_vout, "selected surface pixel format is R5G6B5");
824                     break;
825                 case D3DFMT_X1R5G5B5:
826                     msg_Dbg( p_vout, "selected surface pixel format is X1R5G5B5");
827                     break;
828                 default:
829                     msg_Dbg( p_vout, "selected surface pixel format is 0x%0X", format);
830                     break;
831             }
832             return format;
833         }
834         else if( D3DERR_NOTAVAILABLE != hr )
835         {
836             msg_Err( p_vout, "Could not query adapter supported formats. (hr=0x%lX)", hr);
837             break;
838         }
839     }
840     return D3DFMT_UNKNOWN;
841 }
842
843 static D3DFORMAT Direct3DVoutFindFormat(vout_thread_t *p_vout, int i_chroma, D3DFORMAT target)
844 {
845     //if( p_vout->p_sys->b_hw_yuv && ! _got_vista_or_above )
846     if( p_vout->p_sys->b_hw_yuv )
847     {
848     /* it sounds like vista does not support YUV surfaces at all */
849         switch( i_chroma )
850         {
851             case VLC_CODEC_UYVY:
852             {
853                 static const D3DFORMAT formats[] =
854                     { D3DFMT_UYVY, D3DFMT_YUY2, D3DFMT_X8R8G8B8, D3DFMT_A8R8G8B8, D3DFMT_R5G6B5, D3DFMT_X1R5G5B5 };
855                 return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
856             }
857             case VLC_CODEC_I420:
858             case VLC_CODEC_I422:
859             case VLC_CODEC_YV12:
860             {
861                 /* typically 3D textures don't support planar format
862                 ** fallback to packed version and use CPU for the conversion
863                 */
864                 static const D3DFORMAT formats[] =
865                     { D3DFMT_YUY2, D3DFMT_UYVY, D3DFMT_X8R8G8B8, D3DFMT_A8R8G8B8, D3DFMT_R5G6B5, D3DFMT_X1R5G5B5 };
866                 return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
867             }
868             case VLC_CODEC_YUYV:
869             {
870                 static const D3DFORMAT formats[] =
871                     { D3DFMT_YUY2, D3DFMT_UYVY, D3DFMT_X8R8G8B8, D3DFMT_A8R8G8B8, D3DFMT_R5G6B5, D3DFMT_X1R5G5B5 };
872                 return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
873             }
874         }
875     }
876
877     switch( i_chroma )
878     {
879         case VLC_CODEC_RGB15:
880         {
881             static const D3DFORMAT formats[] =
882                 { D3DFMT_X1R5G5B5 };
883             return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
884         }
885         case VLC_CODEC_RGB16:
886         {
887             static const D3DFORMAT formats[] =
888                 { D3DFMT_R5G6B5 };
889             return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
890         }
891         case VLC_CODEC_RGB24:
892         {
893             static const D3DFORMAT formats[] =
894                 { D3DFMT_R8G8B8, D3DFMT_X8R8G8B8, D3DFMT_A8R8G8B8 };
895             return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
896         }
897         case VLC_CODEC_RGB32:
898         {
899             static const D3DFORMAT formats[] =
900                 { D3DFMT_A8R8G8B8, D3DFMT_X8R8G8B8 };
901             return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
902         }
903         default:
904         {
905             /* use display default format */
906             LPDIRECT3D9 p_d3dobj = p_vout->p_sys->p_d3dobj;
907             D3DDISPLAYMODE d3ddm;
908
909             HRESULT hr = IDirect3D9_GetAdapterDisplayMode(p_d3dobj, D3DADAPTER_DEFAULT, &d3ddm );
910             if( SUCCEEDED(hr))
911             {
912                 /*
913                 ** some professional cards could use some advanced pixel format as default,
914                 ** make sure we stick with chromas that we can handle internally
915                 */
916                 switch( d3ddm.Format )
917                 {
918                     case D3DFMT_R8G8B8:
919                     case D3DFMT_X8R8G8B8:
920                     case D3DFMT_A8R8G8B8:
921                     case D3DFMT_R5G6B5:
922                     case D3DFMT_X1R5G5B5:
923                         msg_Dbg( p_vout, "defaulting to adapter pixel format");
924                         return Direct3DVoutSelectFormat(p_vout, target, &d3ddm.Format, 1);
925                     default:
926                     {
927                         /* if we fall here, that probably means that we need to render some YUV format */
928                         static const D3DFORMAT formats[] =
929                             { D3DFMT_X8R8G8B8, D3DFMT_A8R8G8B8, D3DFMT_R5G6B5, D3DFMT_X1R5G5B5 };
930                         msg_Dbg( p_vout, "defaulting to built-in pixel format");
931                         return Direct3DVoutSelectFormat(p_vout, target, formats, sizeof(formats)/sizeof(D3DFORMAT));
932                     }
933                 }
934             }
935         }
936     }
937     return D3DFMT_UNKNOWN;
938 }
939
940 static int Direct3DVoutSetOutputFormat(vout_thread_t *p_vout, D3DFORMAT format)
941 {
942     switch( format )
943     {
944         case D3DFMT_YUY2:
945             p_vout->output.i_chroma = VLC_CODEC_YUYV;
946             break;
947         case D3DFMT_UYVY:
948             p_vout->output.i_chroma = VLC_CODEC_UYVY;
949             break;
950         case D3DFMT_R8G8B8:
951             p_vout->output.i_chroma = VLC_CODEC_RGB24;
952             p_vout->output.i_rmask = 0xff0000;
953             p_vout->output.i_gmask = 0x00ff00;
954             p_vout->output.i_bmask = 0x0000ff;
955             break;
956         case D3DFMT_X8R8G8B8:
957         case D3DFMT_A8R8G8B8:
958             p_vout->output.i_chroma = VLC_CODEC_RGB32;
959             p_vout->output.i_rmask = 0x00ff0000;
960             p_vout->output.i_gmask = 0x0000ff00;
961             p_vout->output.i_bmask = 0x000000ff;
962             break;
963         case D3DFMT_R5G6B5:
964             p_vout->output.i_chroma = VLC_CODEC_RGB16;
965             p_vout->output.i_rmask = (0x1fL)<<11;
966             p_vout->output.i_gmask = (0x3fL)<<5;
967             p_vout->output.i_bmask = (0x1fL)<<0;
968             break;
969         case D3DFMT_X1R5G5B5:
970             p_vout->output.i_chroma = VLC_CODEC_RGB15;
971             p_vout->output.i_rmask = (0x1fL)<<10;
972             p_vout->output.i_gmask = (0x1fL)<<5;
973             p_vout->output.i_bmask = (0x1fL)<<0;
974             break;
975         default:
976             return VLC_EGENERIC;
977     }
978     return VLC_SUCCESS;
979 }
980
981 /*****************************************************************************
982  * Direct3DVoutCreatePictures: allocate a vector of identical pictures
983  *****************************************************************************
984  * Each picture has an associated offscreen surface in video memory
985  * depending on hardware capabilities the picture chroma will be as close
986  * as possible to the orginal render chroma to reduce CPU conversion overhead
987  * and delegate this work to video card GPU
988  *****************************************************************************/
989 static int Direct3DVoutCreatePictures( vout_thread_t *p_vout, size_t i_num_pics )
990 {
991     LPDIRECT3DDEVICE9       p_d3ddev  = p_vout->p_sys->p_d3ddev;
992     D3DFORMAT               format;
993     HRESULT hr;
994     size_t c;
995     // if vout is already running, use current chroma, otherwise choose from upstream
996     int i_chroma = p_vout->output.i_chroma ? p_vout->output.i_chroma
997                                            : p_vout->render.i_chroma;
998
999     I_OUTPUTPICTURES = 0;
1000
1001     /*
1002     ** find the appropriate D3DFORMAT for the render chroma, the format will be the closest to
1003     ** the requested chroma which is usable by the hardware in an offscreen surface, as they
1004     ** typically support more formats than textures
1005     */
1006     format = Direct3DVoutFindFormat(p_vout, i_chroma, p_vout->p_sys->bbFormat);
1007     if( VLC_SUCCESS != Direct3DVoutSetOutputFormat(p_vout, format) )
1008     {
1009         msg_Err(p_vout, "surface pixel format is not supported.");
1010         return VLC_EGENERIC;
1011     }
1012
1013     for( c=0; c<i_num_pics; )
1014     {
1015
1016         LPDIRECT3DSURFACE9 p_d3dsurf;
1017         picture_t *p_pic = p_vout->p_picture+c;
1018
1019         hr = IDirect3DDevice9_CreateOffscreenPlainSurface(p_d3ddev,
1020                 p_vout->render.i_width,
1021                 p_vout->render.i_height,
1022                 format,
1023                 D3DPOOL_DEFAULT,
1024                 &p_d3dsurf,
1025                 NULL);
1026         if( FAILED(hr) )
1027         {
1028             msg_Err(p_vout, "Failed to create picture surface. (hr=0x%lx)", hr);
1029             Direct3DVoutReleasePictures(p_vout);
1030             return VLC_EGENERIC;
1031         }
1032
1033         /* fill surface with black color */
1034         IDirect3DDevice9_ColorFill(p_d3ddev, p_d3dsurf, NULL, D3DCOLOR_ARGB(0xFF, 0, 0, 0) );
1035
1036         /* assign surface to internal structure */
1037         p_pic->p_sys = (void *)p_d3dsurf;
1038
1039         /* Now that we've got our direct-buffer, we can finish filling in the
1040          * picture_t structures */
1041         switch( p_vout->output.i_chroma )
1042         {
1043             case VLC_CODEC_RGB8:
1044                 p_pic->p->i_lines = p_vout->output.i_height;
1045                 p_pic->p->i_visible_lines = p_vout->output.i_height;
1046                 p_pic->p->i_pixel_pitch = 1;
1047                 p_pic->p->i_visible_pitch = p_vout->output.i_width *
1048                     p_pic->p->i_pixel_pitch;
1049                 p_pic->i_planes = 1;
1050             break;
1051             case VLC_CODEC_RGB15:
1052             case VLC_CODEC_RGB16:
1053                 p_pic->p->i_lines = p_vout->output.i_height;
1054                 p_pic->p->i_visible_lines = p_vout->output.i_height;
1055                 p_pic->p->i_pixel_pitch = 2;
1056                 p_pic->p->i_visible_pitch = p_vout->output.i_width *
1057                     p_pic->p->i_pixel_pitch;
1058                 p_pic->i_planes = 1;
1059             break;
1060             case VLC_CODEC_RGB24:
1061                 p_pic->p->i_lines = p_vout->output.i_height;
1062                 p_pic->p->i_visible_lines = p_vout->output.i_height;
1063                 p_pic->p->i_pixel_pitch = 3;
1064                 p_pic->p->i_visible_pitch = p_vout->output.i_width *
1065                     p_pic->p->i_pixel_pitch;
1066                 p_pic->i_planes = 1;
1067             break;
1068             case VLC_CODEC_RGB32:
1069                 p_pic->p->i_lines = p_vout->output.i_height;
1070                 p_pic->p->i_visible_lines = p_vout->output.i_height;
1071                 p_pic->p->i_pixel_pitch = 4;
1072                 p_pic->p->i_visible_pitch = p_vout->output.i_width *
1073                     p_pic->p->i_pixel_pitch;
1074                 p_pic->i_planes = 1;
1075                 break;
1076             case VLC_CODEC_UYVY:
1077             case VLC_CODEC_YUYV:
1078                 p_pic->p->i_lines = p_vout->output.i_height;
1079                 p_pic->p->i_visible_lines = p_vout->output.i_height;
1080                 p_pic->p->i_pixel_pitch = 2;
1081                 p_pic->p->i_visible_pitch = p_vout->output.i_width *
1082                     p_pic->p->i_pixel_pitch;
1083                 p_pic->i_planes = 1;
1084                 break;
1085             default:
1086                 Direct3DVoutReleasePictures(p_vout);
1087                 return VLC_EGENERIC;
1088         }
1089         p_pic->i_status = DESTROYED_PICTURE;
1090         p_pic->i_type   = DIRECT_PICTURE;
1091         p_pic->b_slow   = true;
1092         p_pic->pf_lock  = Direct3DVoutLockSurface;
1093         p_pic->pf_unlock = Direct3DVoutUnlockSurface;
1094         PP_OUTPUTPICTURE[c] = p_pic;
1095
1096         I_OUTPUTPICTURES = ++c;
1097     }
1098
1099     msg_Dbg( p_vout, "%u Direct3D pictures created successfully", c );
1100
1101     return VLC_SUCCESS;
1102 }
1103
1104 /*****************************************************************************
1105  * Direct3DVoutReleasePictures: destroy a picture vector
1106  *****************************************************************************
1107  * release all video resources used for pictures
1108  *****************************************************************************/
1109 static void Direct3DVoutReleasePictures( vout_thread_t *p_vout)
1110 {
1111     size_t i_num_pics = I_OUTPUTPICTURES;
1112     size_t c;
1113     for( c=0; c<i_num_pics; ++c )
1114     {
1115         picture_t *p_pic = p_vout->p_picture+c;
1116         if( p_pic->p_sys )
1117         {
1118             LPDIRECT3DSURFACE9 p_d3dsurf = (LPDIRECT3DSURFACE9)p_pic->p_sys;
1119
1120             p_pic->p_sys = NULL;
1121
1122             if( p_d3dsurf )
1123             {
1124                 IDirect3DSurface9_Release(p_d3dsurf);
1125             }
1126         }
1127     }
1128     msg_Dbg( p_vout, "%u Direct3D pictures released.", c);
1129
1130     I_OUTPUTPICTURES = 0;
1131 }
1132
1133 /*****************************************************************************
1134  * Direct3DVoutLockSurface: Lock surface and get picture data pointer
1135  *****************************************************************************
1136  * This function locks a surface and get the surface descriptor which amongst
1137  * other things has the pointer to the picture data.
1138  *****************************************************************************/
1139 static int Direct3DVoutLockSurface( vout_thread_t *p_vout, picture_t *p_pic )
1140 {
1141     HRESULT hr;
1142     D3DLOCKED_RECT d3drect;
1143     LPDIRECT3DSURFACE9 p_d3dsurf = (LPDIRECT3DSURFACE9)p_pic->p_sys;
1144
1145     if( NULL == p_d3dsurf )
1146         return VLC_EGENERIC;
1147
1148     /* Lock the surface to get a valid pointer to the picture buffer */
1149     hr = IDirect3DSurface9_LockRect(p_d3dsurf, &d3drect, NULL, 0);
1150     if( FAILED(hr) )
1151     {
1152         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1153         return VLC_EGENERIC;
1154     }
1155
1156     /* fill in buffer info in first plane */
1157     p_pic->p->p_pixels = d3drect.pBits;
1158     p_pic->p->i_pitch  = d3drect.Pitch;
1159
1160     return VLC_SUCCESS;
1161 }
1162
1163 /*****************************************************************************
1164  * Direct3DVoutUnlockSurface: Unlock a surface locked by Direct3DLockSurface().
1165  *****************************************************************************/
1166 static int Direct3DVoutUnlockSurface( vout_thread_t *p_vout, picture_t *p_pic )
1167 {
1168     HRESULT hr;
1169     LPDIRECT3DSURFACE9 p_d3dsurf = (LPDIRECT3DSURFACE9)p_pic->p_sys;
1170
1171     if( NULL == p_d3dsurf )
1172         return VLC_EGENERIC;
1173
1174     /* Unlock the Surface */
1175     hr = IDirect3DSurface9_UnlockRect(p_d3dsurf);
1176     if( FAILED(hr) )
1177     {
1178         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1179         return VLC_EGENERIC;
1180     }
1181     return VLC_SUCCESS;
1182 }
1183
1184 /*****************************************************************************
1185  * Direct3DVoutCreateScene: allocate and initialize a 3D scene
1186  *****************************************************************************
1187  * for advanced blending/filtering a texture needs be used in a 3D scene.
1188  *****************************************************************************/
1189
1190 static int Direct3DVoutCreateScene( vout_thread_t *p_vout )
1191 {
1192     LPDIRECT3DDEVICE9       p_d3ddev  = p_vout->p_sys->p_d3ddev;
1193     LPDIRECT3DTEXTURE9      p_d3dtex;
1194     LPDIRECT3DVERTEXBUFFER9 p_d3dvtc;
1195
1196     HRESULT hr;
1197
1198     /*
1199     ** Create a texture for use when rendering a scene
1200     ** for performance reason, texture format is identical to backbuffer
1201     ** which would usually be a RGB format
1202     */
1203     hr = IDirect3DDevice9_CreateTexture(p_d3ddev,
1204             p_vout->render.i_width,
1205             p_vout->render.i_height,
1206             1,
1207             D3DUSAGE_RENDERTARGET,
1208             p_vout->p_sys->bbFormat,
1209             D3DPOOL_DEFAULT,
1210             &p_d3dtex,
1211             NULL);
1212     if( FAILED(hr))
1213     {
1214         msg_Err(p_vout, "Failed to create texture. (hr=0x%lx)", hr);
1215         return VLC_EGENERIC;
1216     }
1217
1218     /*
1219     ** Create a vertex buffer for use when rendering scene
1220     */
1221     hr = IDirect3DDevice9_CreateVertexBuffer(p_d3ddev,
1222             sizeof(CUSTOMVERTEX)*4,
1223             D3DUSAGE_DYNAMIC|D3DUSAGE_WRITEONLY,
1224             D3DFVF_CUSTOMVERTEX,
1225             D3DPOOL_DEFAULT,
1226             &p_d3dvtc,
1227             NULL);
1228     if( FAILED(hr) )
1229     {
1230         msg_Err(p_vout, "Failed to create vertex buffer. (hr=0x%lx)", hr);
1231             IDirect3DTexture9_Release(p_d3dtex);
1232         return VLC_EGENERIC;
1233     }
1234
1235     p_vout->p_sys->p_d3dtex = p_d3dtex;
1236     p_vout->p_sys->p_d3dvtc = p_d3dvtc;
1237
1238     // Texture coordinates outside the range [0.0, 1.0] are set
1239     // to the texture color at 0.0 or 1.0, respectively.
1240     IDirect3DDevice9_SetSamplerState(p_d3ddev, 0, D3DSAMP_ADDRESSU, D3DTADDRESS_CLAMP);
1241     IDirect3DDevice9_SetSamplerState(p_d3ddev, 0, D3DSAMP_ADDRESSV, D3DTADDRESS_CLAMP);
1242
1243     // Set linear filtering quality
1244     IDirect3DDevice9_SetSamplerState(p_d3ddev, 0, D3DSAMP_MINFILTER, D3DTEXF_LINEAR);
1245     IDirect3DDevice9_SetSamplerState(p_d3ddev, 0, D3DSAMP_MAGFILTER, D3DTEXF_LINEAR);
1246
1247     // set maximum ambient light
1248     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_AMBIENT, D3DCOLOR_XRGB(255,255,255));
1249
1250     // Turn off culling
1251     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_CULLMODE, D3DCULL_NONE);
1252
1253     // Turn off the zbuffer
1254     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_ZENABLE, D3DZB_FALSE);
1255
1256     // Turn off lights
1257     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_LIGHTING, FALSE);
1258
1259     // Enable dithering
1260     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_DITHERENABLE, TRUE);
1261
1262     // disable stencil
1263     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_STENCILENABLE, FALSE);
1264
1265     // manage blending
1266     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_ALPHABLENDENABLE, TRUE);
1267     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_SRCBLEND,D3DBLEND_SRCALPHA);
1268     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_DESTBLEND,D3DBLEND_INVSRCALPHA);
1269     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_ALPHATESTENABLE,TRUE);
1270     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_ALPHAREF, 0x10);
1271     IDirect3DDevice9_SetRenderState(p_d3ddev, D3DRS_ALPHAFUNC,D3DCMP_GREATER);
1272
1273     // Set texture states
1274     IDirect3DDevice9_SetTextureStageState(p_d3ddev, 0, D3DTSS_COLOROP,D3DTOP_MODULATE);
1275     IDirect3DDevice9_SetTextureStageState(p_d3ddev, 0, D3DTSS_COLORARG1,D3DTA_TEXTURE);
1276     IDirect3DDevice9_SetTextureStageState(p_d3ddev, 0, D3DTSS_COLORARG2,D3DTA_DIFFUSE);
1277
1278     // turn off alpha operation
1279     IDirect3DDevice9_SetTextureStageState(p_d3ddev, 0, D3DTSS_ALPHAOP, D3DTOP_DISABLE);
1280
1281     msg_Dbg( p_vout, "Direct3D scene created successfully");
1282
1283     return VLC_SUCCESS;
1284 }
1285
1286 /*****************************************************************************
1287  * Direct3DVoutReleaseScene
1288  *****************************************************************************/
1289 static void Direct3DVoutReleaseScene( vout_thread_t *p_vout )
1290 {
1291     LPDIRECT3DTEXTURE9      p_d3dtex = p_vout->p_sys->p_d3dtex;
1292     LPDIRECT3DVERTEXBUFFER9 p_d3dvtc = p_vout->p_sys->p_d3dvtc;
1293
1294     if( p_d3dvtc )
1295     {
1296         IDirect3DVertexBuffer9_Release(p_d3dvtc);
1297         p_vout->p_sys->p_d3dvtc = NULL;
1298     }
1299
1300     if( p_d3dtex )
1301     {
1302         IDirect3DTexture9_Release(p_d3dtex);
1303         p_vout->p_sys->p_d3dtex = NULL;
1304     }
1305     msg_Dbg( p_vout, "Direct3D scene released successfully");
1306 }
1307
1308 /*****************************************************************************
1309  * Render: copy picture surface into a texture and render into a scene
1310  *****************************************************************************
1311  * This function is intented for higher end 3D cards, with pixel shader support
1312  * and at least 64 MB of video RAM.
1313  *****************************************************************************/
1314 static void Direct3DVoutRenderScene( vout_thread_t *p_vout, picture_t *p_pic )
1315 {
1316     LPDIRECT3DDEVICE9       p_d3ddev  = p_vout->p_sys->p_d3ddev;
1317     LPDIRECT3DTEXTURE9      p_d3dtex;
1318     LPDIRECT3DVERTEXBUFFER9 p_d3dvtc;
1319     LPDIRECT3DSURFACE9      p_d3dsrc, p_d3ddest;
1320     CUSTOMVERTEX            *p_vertices;
1321     HRESULT hr;
1322     float f_width, f_height;
1323
1324     if( p_vout->p_sys->i_changes & SWITCHING_MODE_FLAG )
1325         return;
1326     
1327     // check if device is still available
1328     hr = IDirect3DDevice9_TestCooperativeLevel(p_d3ddev);
1329     if( FAILED(hr) )
1330     {
1331         if( (D3DERR_DEVICENOTRESET != hr)
1332          || (VLC_SUCCESS != Direct3DVoutResetDevice(p_vout)) )
1333         {
1334             // device is not usable at present (lost device, out of video mem ?)
1335             return;
1336         }
1337     }
1338     p_d3dtex  = p_vout->p_sys->p_d3dtex;
1339     p_d3dvtc  = p_vout->p_sys->p_d3dvtc;
1340
1341     /* Clear the backbuffer and the zbuffer */
1342     hr = IDirect3DDevice9_Clear( p_d3ddev, 0, NULL, D3DCLEAR_TARGET,
1343                               D3DCOLOR_XRGB(0, 0, 0), 1.0f, 0 );
1344     if( FAILED(hr) )
1345     {
1346         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1347         return;
1348     }
1349
1350     /*  retrieve picture surface */
1351     p_d3dsrc = (LPDIRECT3DSURFACE9)p_pic->p_sys;
1352     if( NULL == p_d3dsrc )
1353     {
1354         msg_Dbg( p_vout, "no surface to render ?");
1355         return;
1356     }
1357
1358     /* retrieve texture top-level surface */
1359     hr = IDirect3DTexture9_GetSurfaceLevel(p_d3dtex, 0, &p_d3ddest);
1360     if( FAILED(hr) )
1361     {
1362         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1363         return;
1364     }
1365
1366     /* Copy picture surface into texture surface, color space conversion happens here */
1367     hr = IDirect3DDevice9_StretchRect(p_d3ddev, p_d3dsrc, NULL, p_d3ddest, NULL, D3DTEXF_NONE);
1368     IDirect3DSurface9_Release(p_d3ddest);
1369     if( FAILED(hr) )
1370     {
1371         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1372         return;
1373     }
1374
1375     /* Update the vertex buffer */
1376     hr = IDirect3DVertexBuffer9_Lock(p_d3dvtc, 0, 0, (VOID **)(&p_vertices), D3DLOCK_DISCARD);
1377     if( FAILED(hr) )
1378     {
1379         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1380         return;
1381     }
1382
1383     /* Setup vertices */
1384     f_width  = (float)(p_vout->output.i_width);
1385     f_height = (float)(p_vout->output.i_height);
1386
1387     /* -0.5f is a "feature" of DirectX and it seems to apply to Direct3d also */
1388     /* http://www.sjbrown.co.uk/2003/05/01/fix-directx-rasterisation/ */
1389     p_vertices[0].x       = -0.5f;       // left
1390     p_vertices[0].y       = -0.5f;       // top
1391     p_vertices[0].z       = 0.0f;
1392     p_vertices[0].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1393     p_vertices[0].rhw     = 1.0f;
1394     p_vertices[0].tu      = 0.0f;
1395     p_vertices[0].tv      = 0.0f;
1396
1397     p_vertices[1].x       = f_width - 0.5f;    // right
1398     p_vertices[1].y       = -0.5f;       // top
1399     p_vertices[1].z       = 0.0f;
1400     p_vertices[1].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1401     p_vertices[1].rhw     = 1.0f;
1402     p_vertices[1].tu      = 1.0f;
1403     p_vertices[1].tv      = 0.0f;
1404
1405     p_vertices[2].x       = f_width - 0.5f;    // right
1406     p_vertices[2].y       = f_height - 0.5f;   // bottom
1407     p_vertices[2].z       = 0.0f;
1408     p_vertices[2].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1409     p_vertices[2].rhw     = 1.0f;
1410     p_vertices[2].tu      = 1.0f;
1411     p_vertices[2].tv      = 1.0f;
1412
1413     p_vertices[3].x       = -0.5f;       // left
1414     p_vertices[3].y       = f_height - 0.5f;   // bottom
1415     p_vertices[3].z       = 0.0f;
1416     p_vertices[3].diffuse = D3DCOLOR_ARGB(255, 255, 255, 255);
1417     p_vertices[3].rhw     = 1.0f;
1418     p_vertices[3].tu      = 0.0f;
1419     p_vertices[3].tv      = 1.0f;
1420
1421     hr= IDirect3DVertexBuffer9_Unlock(p_d3dvtc);
1422     if( FAILED(hr) )
1423     {
1424         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1425         return;
1426     }
1427
1428     // Begin the scene
1429     hr = IDirect3DDevice9_BeginScene(p_d3ddev);
1430     if( FAILED(hr) )
1431     {
1432         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1433         return;
1434     }
1435
1436     // Setup our texture. Using textures introduces the texture stage states,
1437     // which govern how textures get blended together (in the case of multiple
1438     // textures) and lighting information. In this case, we are modulating
1439     // (blending) our texture with the diffuse color of the vertices.
1440     hr = IDirect3DDevice9_SetTexture(p_d3ddev, 0, (LPDIRECT3DBASETEXTURE9)p_d3dtex);
1441     if( FAILED(hr) )
1442     {
1443         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1444         IDirect3DDevice9_EndScene(p_d3ddev);
1445         return;
1446     }
1447
1448     // Render the vertex buffer contents
1449     hr = IDirect3DDevice9_SetStreamSource(p_d3ddev, 0, p_d3dvtc, 0, sizeof(CUSTOMVERTEX));
1450     if( FAILED(hr) )
1451     {
1452         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1453         IDirect3DDevice9_EndScene(p_d3ddev);
1454         return;
1455     }
1456
1457     // we use FVF instead of vertex shader
1458     hr = IDirect3DDevice9_SetFVF(p_d3ddev, D3DFVF_CUSTOMVERTEX);
1459     if( FAILED(hr) )
1460     {
1461         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1462         IDirect3DDevice9_EndScene(p_d3ddev);
1463         return;
1464     }
1465
1466     // draw rectangle
1467     hr = IDirect3DDevice9_DrawPrimitive(p_d3ddev, D3DPT_TRIANGLEFAN, 0, 2);
1468     if( FAILED(hr) )
1469     {
1470         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1471         IDirect3DDevice9_EndScene(p_d3ddev);
1472         return;
1473     }
1474
1475     // End the scene
1476     hr = IDirect3DDevice9_EndScene(p_d3ddev);
1477     if( FAILED(hr) )
1478     {
1479         msg_Dbg( p_vout, "%s:%d (hr=0x%0lX)", __FUNCTION__, __LINE__, hr);
1480         return;
1481     }
1482 }
1483
1484
1485 /*****************************************************************************
1486  * DesktopCallback: desktop mode variable callback
1487  *****************************************************************************/
1488 static int DesktopCallback( vlc_object_t *p_this, char const *psz_cmd,
1489                             vlc_value_t oldval, vlc_value_t newval,
1490                             void *p_data )
1491 {
1492     VLC_UNUSED( psz_cmd );
1493     VLC_UNUSED( oldval );
1494     VLC_UNUSED( p_data );
1495
1496     vout_thread_t *p_vout = (vout_thread_t *)p_this;
1497
1498     if( (newval.b_bool && !p_vout->p_sys->b_desktop) ||
1499         (!newval.b_bool && p_vout->p_sys->b_desktop) )
1500     {
1501         playlist_t *p_playlist = pl_Hold( p_vout );
1502
1503         if( p_playlist )
1504         {
1505             /* Modify playlist as well because the vout might have to be
1506              * restarted */
1507             var_Create( p_playlist, "direct3d-desktop", VLC_VAR_BOOL );
1508             var_Set( p_playlist, "direct3d-desktop", newval );
1509             pl_Release( p_vout );
1510         }
1511
1512         p_vout->p_sys->i_changes |= DX_DESKTOP_CHANGE;
1513     }
1514
1515     return VLC_SUCCESS;
1516 }