direct3d11: fix crash when resource allocation fails
[vlc.git] / modules / video_output / win32 / direct3d11.c
1 /*****************************************************************************
2  * direct3d11.c: Windows Direct3D11 video output module
3  *****************************************************************************
4  * Copyright (C) 2014-2015 VLC authors and VideoLAN
5  *
6  * Authors: Martell Malone <martellmalone@gmail.com>
7  *          Steve Lhomme <robux4@gmail.com>
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU Lesser General Public License as published by
11  * the Free Software Foundation; either version 2.1 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public License
20  * along with this program; if not, write to the Free Software Foundation,
21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 #ifdef HAVE_CONFIG_H
25 # include "config.h"
26 #endif
27
28 #include <vlc_common.h>
29 #include <vlc_plugin.h>
30 #include <vlc_vout_display.h>
31
32 #include <assert.h>
33 #include <math.h>
34
35 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < _WIN32_WINNT_WIN7
36 # undef _WIN32_WINNT
37 # define _WIN32_WINNT _WIN32_WINNT_WIN7
38 #endif
39
40 #define COBJMACROS
41 #include <initguid.h>
42 #include <d3d11.h>
43 #ifdef HAVE_DXGI1_6_H
44 # include <dxgi1_6.h>
45 #else
46 # include <dxgi1_5.h>
47 #endif
48
49 /* avoided until we can pass ISwapchainPanel without c++/cx mode
50 # include <windows.ui.xaml.media.dxinterop.h> */
51
52 #include "../../video_chroma/d3d11_fmt.h"
53 #include "d3d11_quad.h"
54 #include "d3d11_shaders.h"
55
56 #include "common.h"
57
58 DEFINE_GUID(GUID_SWAPCHAIN_WIDTH,  0xf1b59347, 0x1643, 0x411a, 0xad, 0x6b, 0xc7, 0x80, 0x17, 0x7a, 0x06, 0xb6);
59 DEFINE_GUID(GUID_SWAPCHAIN_HEIGHT, 0x6ea976a0, 0x9d60, 0x4bb7, 0xa5, 0xa9, 0x7d, 0xd1, 0x18, 0x7f, 0xc9, 0xbd);
60
61 static int  Open(vlc_object_t *);
62 static void Close(vlc_object_t *);
63
64 #define D3D11_HELP N_("Recommended video output for Windows 8 and later versions")
65 #define HW_BLENDING_TEXT N_("Use hardware blending support")
66 #define HW_BLENDING_LONGTEXT N_(\
67     "Try to use hardware acceleration for subtitle/OSD blending.")
68
69 vlc_module_begin ()
70     set_shortname("Direct3D11")
71     set_description(N_("Direct3D11 video output"))
72     set_help(D3D11_HELP)
73     set_category(CAT_VIDEO)
74     set_subcategory(SUBCAT_VIDEO_VOUT)
75
76     add_bool("direct3d11-hw-blending", true, HW_BLENDING_TEXT, HW_BLENDING_LONGTEXT, true)
77
78 #if VLC_WINSTORE_APP
79     add_integer("winrt-d3dcontext",    0x0, NULL, NULL, true); /* ID3D11DeviceContext* */
80     add_integer("winrt-swapchain",     0x0, NULL, NULL, true); /* IDXGISwapChain1*     */
81 #endif
82
83     set_capability("vout display", 300)
84     add_shortcut("direct3d11")
85     set_callbacks(Open, Close)
86 vlc_module_end ()
87
88 struct vout_display_sys_t
89 {
90     vout_display_sys_win32_t sys;
91
92     display_info_t           display;
93
94 #if !VLC_WINSTORE_APP
95     HINSTANCE                hdxgi_dll;        /* handle of the opened dxgi dll */
96     d3d11_handle_t           hd3d;
97 #endif
98     IDXGISwapChain1          *dxgiswapChain;   /* DXGI 1.2 swap chain */
99     IDXGISwapChain4          *dxgiswapChain4;  /* DXGI 1.5 for HDR */
100     d3d11_device_t           d3d_dev;
101     d3d_quad_t               picQuad;
102
103     picture_sys_t            stagingSys;
104
105     ID3D11RenderTargetView   *d3drenderTargetView[D3D11_MAX_SHADER_VIEW];
106
107     ID3D11InputLayout         *pVertexLayout;
108     ID3D11VertexShader        *flatVSShader;
109     ID3D11VertexShader        *projectionVSShader;
110
111     /* copy from the decoder pool into picSquad before display
112      * Uses a Texture2D with slices rather than a Texture2DArray for the decoder */
113     bool                     legacy_shader;
114
115     // SPU
116     vlc_fourcc_t             pSubpictureChromas[2];
117     ID3D11PixelShader        *pSPUPixelShader[D3D11_MAX_SHADER_VIEW];
118     const d3d_format_t       *d3dregion_format;
119     int                      d3dregion_count;
120     picture_t                **d3dregions;
121 };
122
123 #define RECTWidth(r)   (int)((r).right - (r).left)
124 #define RECTHeight(r)  (int)((r).bottom - (r).top)
125
126 static picture_pool_t *Pool(vout_display_t *, unsigned);
127
128 static void Prepare(vout_display_t *, picture_t *, subpicture_t *subpicture, vlc_tick_t);
129 static void Display(vout_display_t *, picture_t *, subpicture_t *subpicture);
130
131 static void Direct3D11Destroy(vout_display_t *);
132
133 static int  Direct3D11Open (vout_display_t *);
134 static void Direct3D11Close(vout_display_t *);
135
136 static int SetupOutputFormat(vout_display_t *, video_format_t *);
137 static int  Direct3D11CreateFormatResources (vout_display_t *, const video_format_t *);
138 static int  Direct3D11CreateGenericResources(vout_display_t *);
139 static void Direct3D11DestroyResources(vout_display_t *);
140
141 static void Direct3D11DestroyPool(vout_display_t *);
142
143 static void DestroyDisplayPoolPicture(picture_t *);
144 static void Direct3D11DeleteRegions(int, picture_t **);
145 static int Direct3D11MapSubpicture(vout_display_t *, int *, picture_t ***, subpicture_t *);
146
147 static void SetQuadVSProjection(vout_display_t *, d3d_quad_t *, const vlc_viewpoint_t *);
148 static void UpdatePicQuadPosition(vout_display_t *);
149
150 static int Control(vout_display_t *, int, va_list);
151
152 static int Direct3D11MapPoolTexture(picture_t *picture)
153 {
154     picture_sys_t *p_sys = picture->p_sys;
155     D3D11_MAPPED_SUBRESOURCE mappedResource;
156     HRESULT hr;
157
158     ID3D11Device *dev;
159     ID3D11DeviceContext_GetDevice(p_sys->context, &dev);
160     ID3D11Device_Release(dev);
161
162 #ifndef NDEBUG
163     D3D11_TEXTURE2D_DESC dsc;
164     ID3D11Texture2D_GetDesc(p_sys->texture[KNOWN_DXGI_INDEX], &dsc);
165     assert(dsc.CPUAccessFlags & D3D11_CPU_ACCESS_WRITE);
166     assert(dsc.Usage & D3D11_USAGE_DYNAMIC);
167 #endif
168
169     hr = ID3D11DeviceContext_Map(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], p_sys->slice_index, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
170     if( FAILED(hr) )
171     {
172         return VLC_EGENERIC;
173     }
174     return CommonUpdatePicture(picture, NULL, mappedResource.pData, mappedResource.RowPitch);
175 }
176
177 static void Direct3D11UnmapPoolTexture(picture_t *picture)
178 {
179     picture_sys_t *p_sys = picture->p_sys;
180     ID3D11DeviceContext_Unmap(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0);
181 }
182
183 #if !VLC_WINSTORE_APP
184 static int OpenHwnd(vout_display_t *vd)
185 {
186     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
187     if (!sys)
188         return VLC_ENOMEM;
189
190     return D3D11_Create(vd, &sys->hd3d, true);
191 }
192 #else
193 static int OpenCoreW(vout_display_t *vd)
194 {
195     IDXGISwapChain1* dxgiswapChain  = var_InheritInteger(vd, "winrt-swapchain");
196     if (!dxgiswapChain)
197         return VLC_EGENERIC;
198     ID3D11DeviceContext* d3dcontext = var_InheritInteger(vd, "winrt-d3dcontext");
199     if (!d3dcontext)
200         return VLC_EGENERIC;
201     ID3D11Device* d3ddevice = NULL;
202     ID3D11DeviceContext_GetDevice(d3dcontext, &d3ddevice);
203     if (!d3ddevice)
204         return VLC_EGENERIC;
205
206     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
207     if (!sys)
208         return VLC_ENOMEM;
209
210     sys->dxgiswapChain = dxgiswapChain;
211     sys->d3d_dev.d3ddevice     = d3ddevice;
212     sys->d3d_dev.d3dcontext    = d3dcontext;
213     sys->d3d_dev.feature_level = ID3D11Device_GetFeatureLevel(sys->d3d_dev.d3ddevice );
214     IDXGISwapChain_AddRef     (sys->dxgiswapChain);
215     ID3D11Device_AddRef       (sys->d3d_dev.d3ddevice);
216     ID3D11DeviceContext_AddRef(sys->d3d_dev.d3dcontext);
217
218     return VLC_SUCCESS;
219 }
220 #endif
221
222 #if VLC_WINSTORE_APP
223 static bool GetRect(const vout_display_sys_win32_t *p_sys, RECT *out)
224 {
225     const vout_display_sys_t *sys = (const vout_display_sys_t *)p_sys;
226     out->left   = 0;
227     out->top    = 0;
228     uint32_t i_width;
229     uint32_t i_height;
230     UINT dataSize = sizeof(i_width);
231     HRESULT hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_WIDTH, &dataSize, &i_width);
232     if (FAILED(hr)) {
233         return false;
234     }
235     dataSize = sizeof(i_height);
236     hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_HEIGHT, &dataSize, &i_height);
237     if (FAILED(hr)) {
238         return false;
239     }
240     out->right  = i_width;
241     out->bottom = i_height;
242     return true;
243 }
244 #endif
245
246 static unsigned int GetPictureWidth(const vout_display_t *vd)
247 {
248     return vd->sys->picQuad.i_width;
249 }
250
251 static unsigned int GetPictureHeight(const vout_display_t *vd)
252 {
253     return vd->sys->picQuad.i_height;
254 }
255
256 static int Open(vlc_object_t *object)
257 {
258     vout_display_t *vd = (vout_display_t *)object;
259
260 #if !VLC_WINSTORE_APP
261     /* Allow using D3D11 automatically starting from Windows 8.1 */
262     if (!vd->obj.force)
263     {
264         bool isWin81OrGreater = false;
265         HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32.dll"));
266         if (likely(hKernel32 != NULL))
267             isWin81OrGreater = GetProcAddress(hKernel32, "IsProcessCritical") != NULL;
268         if (!isWin81OrGreater)
269             return VLC_EGENERIC;
270     }
271 #endif
272
273 #if !VLC_WINSTORE_APP
274     int ret = OpenHwnd(vd);
275 #else
276     int ret = OpenCoreW(vd);
277 #endif
278
279     if (ret != VLC_SUCCESS)
280         return ret;
281
282     if (CommonInit(vd))
283         goto error;
284
285 #if VLC_WINSTORE_APP
286     vd->sys->sys.pf_GetRect = GetRect;
287 #endif
288     vd->sys->sys.pf_GetPictureWidth  = GetPictureWidth;
289     vd->sys->sys.pf_GetPictureHeight = GetPictureHeight;
290
291     if (Direct3D11Open(vd)) {
292         msg_Err(vd, "Direct3D11 could not be opened");
293         goto error;
294     }
295
296 #if !VLC_WINSTORE_APP
297     EventThreadUpdateTitle(vd->sys->sys.event, VOUT_TITLE " (Direct3D11 output)");
298 #endif
299     msg_Dbg(vd, "Direct3D11 device adapter successfully initialized");
300
301     vd->info.has_double_click     = true;
302     vd->info.has_pictures_invalid = vd->info.is_slow;
303
304     if (var_InheritBool(vd, "direct3d11-hw-blending") &&
305         vd->sys->d3dregion_format != NULL)
306     {
307         vd->sys->pSubpictureChromas[0] = vd->sys->d3dregion_format->fourcc;
308         vd->sys->pSubpictureChromas[1] = 0;
309         vd->info.subpicture_chromas = vd->sys->pSubpictureChromas;
310     }
311     else
312         vd->info.subpicture_chromas = NULL;
313
314     vd->pool    = Pool;
315     vd->prepare = Prepare;
316     vd->display = Display;
317     vd->control = Control;
318
319     msg_Dbg(vd, "Direct3D11 Open Succeeded");
320
321     return VLC_SUCCESS;
322
323 error:
324     Close(object);
325     return VLC_EGENERIC;
326 }
327
328 static void Close(vlc_object_t *object)
329 {
330     vout_display_t * vd = (vout_display_t *)object;
331
332     Direct3D11Close(vd);
333     CommonClean(vd);
334     Direct3D11Destroy(vd);
335     free(vd->sys);
336 }
337
338 static picture_pool_t *Pool(vout_display_t *vd, unsigned pool_size)
339 {
340     /* compensate for extra hardware decoding pulling extra pictures from our pool */
341     pool_size += 2;
342
343     vout_display_sys_t *sys = vd->sys;
344     picture_t **pictures = NULL;
345     picture_t *picture;
346     unsigned  picture_count = 0;
347
348     if (sys->sys.pool)
349         return sys->sys.pool;
350
351     if (vd->info.is_slow)
352         pool_size = 1;
353
354     video_format_t surface_fmt = vd->fmt;
355     surface_fmt.i_width  = sys->picQuad.i_width;
356     surface_fmt.i_height = sys->picQuad.i_height;
357
358     if (D3D11_SetupQuad( vd, &sys->d3d_dev, &surface_fmt, &sys->picQuad, &sys->display, &sys->sys.rect_src_clipped,
359                    vd->fmt.projection_mode == PROJECTION_MODE_RECTANGULAR ? sys->flatVSShader : sys->projectionVSShader,
360                    sys->pVertexLayout,
361                    surface_fmt.projection_mode, vd->fmt.orientation ) != VLC_SUCCESS) {
362         msg_Err(vd, "Could not Create the main quad picture.");
363         return NULL;
364     }
365
366     if ( vd->fmt.projection_mode == PROJECTION_MODE_EQUIRECTANGULAR ||
367          vd->fmt.projection_mode == PROJECTION_MODE_CUBEMAP_LAYOUT_STANDARD )
368         SetQuadVSProjection( vd, &sys->picQuad, &vd->cfg->viewpoint );
369
370     if (!vd->info.is_slow) {
371         HRESULT           hr;
372         ID3D10Multithread *pMultithread;
373         hr = ID3D11Device_QueryInterface( sys->d3d_dev.d3ddevice, &IID_ID3D10Multithread, (void **)&pMultithread);
374         if (SUCCEEDED(hr)) {
375             ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
376             ID3D10Multithread_Release(pMultithread);
377         }
378     }
379
380     if (sys->picQuad.formatInfo->formatTexture == DXGI_FORMAT_UNKNOWN)
381         sys->sys.pool = picture_pool_NewFromFormat( &surface_fmt, pool_size );
382     else
383     {
384         ID3D11Texture2D  *textures[pool_size * D3D11_MAX_SHADER_VIEW];
385         memset(textures, 0, sizeof(textures));
386         unsigned slices = pool_size;
387         if (!CanUseVoutPool(&sys->d3d_dev, pool_size))
388             /* only provide enough for the filters, we can still do direct rendering */
389             slices = __MIN(slices, 6);
390
391         if (AllocateTextures(vd, &sys->d3d_dev, sys->picQuad.formatInfo, &surface_fmt, slices, textures))
392             goto error;
393
394         pictures = calloc(pool_size, sizeof(*pictures));
395         if (!pictures)
396             goto error;
397
398         for (picture_count = 0; picture_count < pool_size; picture_count++) {
399             picture_sys_t *picsys = calloc(1, sizeof(*picsys));
400             if (unlikely(picsys == NULL))
401                 goto error;
402
403             for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
404                 picsys->texture[plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW + plane];
405
406             picsys->slice_index = picture_count;
407             picsys->formatTexture = sys->picQuad.formatInfo->formatTexture;
408             picsys->context = sys->d3d_dev.d3dcontext;
409
410             picture_resource_t resource = {
411                 .p_sys = picsys,
412                 .pf_destroy = DestroyDisplayPoolPicture,
413             };
414
415             picture = picture_NewFromResource(&surface_fmt, &resource);
416             if (unlikely(picture == NULL)) {
417                 free(picsys);
418                 msg_Err( vd, "Failed to create picture %d in the pool.", picture_count );
419                 goto error;
420             }
421
422             pictures[picture_count] = picture;
423             /* each picture_t holds a ref to the context and release it on Destroy */
424             ID3D11DeviceContext_AddRef(picsys->context);
425         }
426
427 #ifdef HAVE_ID3D11VIDEODECODER
428         if (is_d3d11_opaque(surface_fmt.i_chroma) && !sys->legacy_shader)
429 #endif
430         {
431             for (picture_count = 0; picture_count < pool_size; picture_count++) {
432                 picture_sys_t *p_sys = pictures[picture_count]->p_sys;
433                 if (!p_sys->texture[0])
434                     continue;
435                 if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->picQuad.formatInfo,
436                                              p_sys->texture, picture_count,
437                                              p_sys->resourceView))
438                     goto error;
439             }
440         }
441
442         picture_pool_configuration_t pool_cfg = {
443             .picture       = pictures,
444             .picture_count = pool_size,
445         };
446         if (vd->info.is_slow && !is_d3d11_opaque(surface_fmt.i_chroma)) {
447             pool_cfg.lock          = Direct3D11MapPoolTexture;
448             //pool_cfg.unlock        = Direct3D11UnmapPoolTexture;
449         }
450         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
451     }
452
453 error:
454     if (sys->sys.pool == NULL) {
455         picture_pool_configuration_t pool_cfg = {
456             .picture_count = 0,
457         };
458         if (pictures) {
459             msg_Dbg(vd, "Failed to create the picture d3d11 pool");
460             for (unsigned i=0;i<picture_count; ++i)
461                 picture_Release(pictures[i]);
462             free(pictures);
463         }
464
465         /* create an empty pool to avoid crashing */
466         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
467     } else {
468         msg_Dbg(vd, "D3D11 pool succeed with %d surfaces (%dx%d) context 0x%p",
469                 pool_size, surface_fmt.i_width, surface_fmt.i_height, sys->d3d_dev.d3dcontext);
470     }
471     return sys->sys.pool;
472 }
473
474 static void DestroyDisplayPoolPicture(picture_t *picture)
475 {
476     picture_sys_t *p_sys = picture->p_sys;
477     ReleasePictureSys( p_sys );
478     free(p_sys);
479     free(picture);
480 }
481
482 #if !VLC_WINSTORE_APP
483 static void FillSwapChainDesc(vout_display_t *vd, DXGI_SWAP_CHAIN_DESC1 *out)
484 {
485     ZeroMemory(out, sizeof(*out));
486     out->BufferCount = 3;
487     out->BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
488     out->SampleDesc.Count = 1;
489     out->SampleDesc.Quality = 0;
490     out->Width = vd->source.i_visible_width;
491     out->Height = vd->source.i_visible_height;
492     out->Format = vd->sys->display.pixelFormat->formatTexture;
493     //out->Flags = 512; // DXGI_SWAP_CHAIN_FLAG_YUV_VIDEO;
494     out->SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
495 }
496 #endif
497
498 static HRESULT UpdateBackBuffer(vout_display_t *vd)
499 {
500     vout_display_sys_t *sys = vd->sys;
501     HRESULT hr;
502     ID3D11Texture2D* pBackBuffer;
503     RECT rect;
504 #if VLC_WINSTORE_APP
505     if (!GetRect(&sys->sys, &rect))
506 #endif
507         rect = sys->sys.rect_dest_clipped;
508     uint32_t i_width = RECTWidth(rect);
509     uint32_t i_height = RECTHeight(rect);
510     D3D11_TEXTURE2D_DESC dsc = { 0 };
511
512     if (sys->d3drenderTargetView[0]) {
513         ID3D11Resource *res = NULL;
514         ID3D11RenderTargetView_GetResource(sys->d3drenderTargetView[0], &res);
515         if (res)
516         {
517             ID3D11Texture2D_GetDesc((ID3D11Texture2D*) res, &dsc);
518             ID3D11Resource_Release(res);
519         }
520     }
521
522     if (dsc.Width == i_width && dsc.Height == i_height)
523         return S_OK; /* nothing changed */
524
525     for (size_t i=0; i < D3D11_MAX_SHADER_VIEW; i++)
526     {
527         if (sys->d3drenderTargetView[i]) {
528             ID3D11RenderTargetView_Release(sys->d3drenderTargetView[i]);
529             sys->d3drenderTargetView[i] = NULL;
530         }
531     }
532
533     /* TODO detect is the size is the same as the output and switch to fullscreen mode */
534     hr = IDXGISwapChain_ResizeBuffers(sys->dxgiswapChain, 0, i_width, i_height,
535         DXGI_FORMAT_UNKNOWN, 0);
536     if (FAILED(hr)) {
537        msg_Err(vd, "Failed to resize the backbuffer. (hr=0x%lX)", hr);
538        return hr;
539     }
540
541     hr = IDXGISwapChain_GetBuffer(sys->dxgiswapChain, 0, &IID_ID3D11Texture2D, (LPVOID *)&pBackBuffer);
542     if (FAILED(hr)) {
543        msg_Err(vd, "Could not get the backbuffer for the Swapchain. (hr=0x%lX)", hr);
544        return hr;
545     }
546
547     hr = D3D11_CreateRenderTargets( &sys->d3d_dev, (ID3D11Resource *)pBackBuffer,
548                                     sys->display.pixelFormat, sys->d3drenderTargetView );
549     ID3D11Texture2D_Release(pBackBuffer);
550     if (FAILED(hr)) {
551         msg_Err(vd, "Failed to create the target view. (hr=0x%lX)", hr);
552         return hr;
553     }
554
555     D3D11_ClearRenderTargets( &sys->d3d_dev, sys->display.pixelFormat, sys->d3drenderTargetView );
556
557     return S_OK;
558 }
559
560 /* rotation around the Z axis */
561 static void getZRotMatrix(float theta, FLOAT matrix[static 16])
562 {
563     float st, ct;
564
565     sincosf(theta, &st, &ct);
566
567     const FLOAT m[] = {
568     /*  x    y    z    w */
569         ct,  -st, 0.f, 0.f,
570         st,  ct,  0.f, 0.f,
571         0.f, 0.f, 1.f, 0.f,
572         0.f, 0.f, 0.f, 1.f
573     };
574
575     memcpy(matrix, m, sizeof(m));
576 }
577
578 /* rotation around the Y axis */
579 static void getYRotMatrix(float theta, FLOAT matrix[static 16])
580 {
581     float st, ct;
582
583     sincosf(theta, &st, &ct);
584
585     const FLOAT m[] = {
586     /*  x    y    z    w */
587         ct,  0.f, -st, 0.f,
588         0.f, 1.f, 0.f, 0.f,
589         st,  0.f, ct,  0.f,
590         0.f, 0.f, 0.f, 1.f
591     };
592
593     memcpy(matrix, m, sizeof(m));
594 }
595
596 /* rotation around the X axis */
597 static void getXRotMatrix(float phi, FLOAT matrix[static 16])
598 {
599     float sp, cp;
600
601     sincosf(phi, &sp, &cp);
602
603     const FLOAT m[] = {
604     /*  x    y    z    w */
605         1.f, 0.f, 0.f, 0.f,
606         0.f, cp,  sp,  0.f,
607         0.f, -sp, cp,  0.f,
608         0.f, 0.f, 0.f, 1.f
609     };
610
611     memcpy(matrix, m, sizeof(m));
612 }
613
614 static void getZoomMatrix(float zoom, FLOAT matrix[static 16]) {
615
616     const FLOAT m[] = {
617         /* x   y     z     w */
618         1.0f, 0.0f, 0.0f, 0.0f,
619         0.0f, 1.0f, 0.0f, 0.0f,
620         0.0f, 0.0f, 1.0f, 0.0f,
621         0.0f, 0.0f, zoom, 1.0f
622     };
623
624     memcpy(matrix, m, sizeof(m));
625 }
626
627 /* perspective matrix see https://www.opengl.org/sdk/docs/man2/xhtml/gluPerspective.xml */
628 static void getProjectionMatrix(float sar, float fovy, FLOAT matrix[static 16]) {
629
630     float zFar  = 1000;
631     float zNear = 0.01;
632
633     float f = 1.f / tanf(fovy / 2.f);
634
635     const FLOAT m[] = {
636         f / sar, 0.f,                   0.f,                0.f,
637         0.f,     f,                     0.f,                0.f,
638         0.f,     0.f,     (zNear + zFar) / (zNear - zFar), -1.f,
639         0.f,     0.f, (2 * zNear * zFar) / (zNear - zFar),  0.f};
640
641      memcpy(matrix, m, sizeof(m));
642 }
643
644 static float UpdateFOVy(float f_fovx, float f_sar)
645 {
646     return 2 * atanf(tanf(f_fovx / 2) / f_sar);
647 }
648
649 static float UpdateZ(float f_fovx, float f_fovy)
650 {
651     /* Do trigonometry to calculate the minimal z value
652      * that will allow us to zoom out without seeing the outside of the
653      * sphere (black borders). */
654     float tan_fovx_2 = tanf(f_fovx / 2);
655     float tan_fovy_2 = tanf(f_fovy / 2);
656     float z_min = - SPHERE_RADIUS / sinf(atanf(sqrtf(
657                     tan_fovx_2 * tan_fovx_2 + tan_fovy_2 * tan_fovy_2)));
658
659     /* The FOV value above which z is dynamically calculated. */
660     const float z_thresh = 90.f;
661
662     float f_z;
663     if (f_fovx <= z_thresh * M_PI / 180)
664         f_z = 0;
665     else
666     {
667         float f = z_min / ((FIELD_OF_VIEW_DEGREES_MAX - z_thresh) * M_PI / 180);
668         f_z = f * f_fovx - f * z_thresh * M_PI / 180;
669         if (f_z < z_min)
670             f_z = z_min;
671     }
672     return f_z;
673 }
674
675 static void SetQuadVSProjection(vout_display_t *vd, d3d_quad_t *quad, const vlc_viewpoint_t *p_vp)
676 {
677     if (!quad->pVertexShaderConstants)
678         return;
679
680 #define RAD(d) ((float) ((d) * M_PI / 180.f))
681     float f_fovx = RAD(p_vp->fov);
682     if ( f_fovx > FIELD_OF_VIEW_DEGREES_MAX * M_PI / 180 + 0.001f ||
683          f_fovx < -0.001f )
684         return;
685
686     float f_sar = (float) vd->cfg->display.width / vd->cfg->display.height;
687     float f_teta = RAD(p_vp->yaw) - (float) M_PI_2;
688     float f_phi  = RAD(p_vp->pitch);
689     float f_roll = RAD(p_vp->roll);
690     float f_fovy = UpdateFOVy(f_fovx, f_sar);
691     float f_z = UpdateZ(f_fovx, f_fovy);
692
693     vout_display_sys_t *sys = vd->sys;
694     HRESULT hr;
695     D3D11_MAPPED_SUBRESOURCE mapped;
696     hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped);
697     if (SUCCEEDED(hr)) {
698         VS_PROJECTION_CONST *dst_data = mapped.pData;
699         getXRotMatrix(f_phi, dst_data->RotX);
700         getYRotMatrix(f_teta,   dst_data->RotY);
701         getZRotMatrix(f_roll,  dst_data->RotZ);
702         getZoomMatrix(SPHERE_RADIUS * f_z, dst_data->View);
703         getProjectionMatrix(f_sar, f_fovy, dst_data->Projection);
704     }
705     ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0);
706 #undef RAD
707 }
708
709 static void UpdateSize(vout_display_t *vd)
710 {
711     vout_display_sys_t *sys = vd->sys;
712     msg_Dbg(vd, "Detected size change %dx%d", RECTWidth(sys->sys.rect_dest_clipped),
713             RECTHeight(sys->sys.rect_dest_clipped));
714
715     UpdateBackBuffer(vd);
716
717     d3d11_device_lock( &sys->d3d_dev );
718
719     UpdatePicQuadPosition(vd);
720
721     D3D11_UpdateQuadPosition(vd, &sys->d3d_dev, &sys->picQuad, &sys->sys.rect_src_clipped,
722                              vd->fmt.orientation);
723
724     d3d11_device_unlock( &sys->d3d_dev );
725 }
726
727 static inline bool RectEquals(const RECT *r1, const RECT *r2)
728 {
729     return r1->bottom == r2->bottom && r1->top == r2->top &&
730            r1->left == r2->left && r1->right == r2->right;
731 }
732
733 static int Control(vout_display_t *vd, int query, va_list args)
734 {
735     vout_display_sys_t *sys = vd->sys;
736     RECT before_src_clipped  = sys->sys.rect_src_clipped;
737     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
738     RECT before_dest         = sys->sys.rect_dest;
739
740     int res = CommonControl( vd, query, args );
741
742     if (query == VOUT_DISPLAY_CHANGE_VIEWPOINT)
743     {
744         const vout_display_cfg_t *cfg = va_arg(args, const vout_display_cfg_t*);
745         if ( sys->picQuad.pVertexShaderConstants )
746         {
747             SetQuadVSProjection( vd, &sys->picQuad, &cfg->viewpoint );
748             res = VLC_SUCCESS;
749         }
750     }
751
752     if (!RectEquals(&before_src_clipped,  &sys->sys.rect_src_clipped) ||
753         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
754         !RectEquals(&before_dest,         &sys->sys.rect_dest) )
755     {
756         UpdateSize(vd);
757     }
758
759     return res;
760 }
761
762 static void Manage(vout_display_t *vd)
763 {
764     vout_display_sys_t *sys = vd->sys;
765     RECT before_src_clipped  = sys->sys.rect_src_clipped;
766     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
767     RECT before_dest         = sys->sys.rect_dest;
768
769     CommonManage(vd);
770
771     if (!RectEquals(&before_src_clipped, &sys->sys.rect_src_clipped) ||
772         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
773         !RectEquals(&before_dest, &sys->sys.rect_dest))
774     {
775         UpdateSize(vd);
776     }
777 }
778
779 static void Prepare(vout_display_t *vd, picture_t *picture,
780                     subpicture_t *subpicture, vlc_tick_t date)
781 {
782     Manage(vd);
783     VLC_UNUSED(date);
784     vout_display_sys_t *sys = vd->sys;
785
786     if (sys->picQuad.formatInfo->formatTexture == DXGI_FORMAT_UNKNOWN)
787     {
788         D3D11_MAPPED_SUBRESOURCE mappedResource;
789         D3D11_TEXTURE2D_DESC texDesc;
790         int i;
791         HRESULT hr;
792         plane_t planes[PICTURE_PLANE_MAX];
793
794         bool b_mapped = true;
795         for (i = 0; i < picture->i_planes; i++) {
796             hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i],
797                                          0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
798             if( unlikely(FAILED(hr)) )
799             {
800                 while (i-- > 0)
801                     ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i], 0);
802                 b_mapped = false;
803                 break;
804             }
805             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[i], &texDesc);
806             planes[i].i_lines = texDesc.Height;
807             planes[i].i_pitch = mappedResource.RowPitch;
808             planes[i].p_pixels = mappedResource.pData;
809
810             planes[i].i_visible_lines = picture->p[i].i_visible_lines;
811             planes[i].i_visible_pitch = picture->p[i].i_visible_pitch;
812         }
813
814         if (b_mapped)
815         {
816             for (i = 0; i < picture->i_planes; i++)
817                 plane_CopyPixels(&planes[i], &picture->p[i]);
818
819             for (i = 0; i < picture->i_planes; i++)
820                 ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i], 0);
821         }
822     }
823     else
824     {
825         picture_sys_t *p_sys = ActivePictureSys(picture);
826
827         if (is_d3d11_opaque(picture->format.i_chroma))
828             d3d11_device_lock( &sys->d3d_dev );
829
830         if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader) {
831             D3D11_TEXTURE2D_DESC srcDesc,texDesc;
832             if (!is_d3d11_opaque(picture->format.i_chroma))
833                 Direct3D11UnmapPoolTexture(picture);
834             ID3D11Texture2D_GetDesc(p_sys->texture[KNOWN_DXGI_INDEX], &srcDesc);
835             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[0], &texDesc);
836             D3D11_BOX box = {
837                 .top = 0,
838                 .bottom = __MIN(srcDesc.Height, texDesc.Height),
839                 .left = 0,
840                 .right = __MIN(srcDesc.Width, texDesc.Width),
841                 .back = 1,
842             };
843             ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev.d3dcontext,
844                                                       sys->stagingSys.resource[KNOWN_DXGI_INDEX],
845                                                       0, 0, 0, 0,
846                                                       p_sys->resource[KNOWN_DXGI_INDEX],
847                                                       p_sys->slice_index, &box);
848         }
849         else
850         {
851             D3D11_TEXTURE2D_DESC texDesc;
852             ID3D11Texture2D_GetDesc(p_sys->texture[0], &texDesc);
853             if (texDesc.BindFlags & D3D11_BIND_SHADER_RESOURCE)
854             {
855                 /* for performance reason we don't want to allocate this during
856                  * display, do it preferrably when creating the texture */
857                 assert(p_sys->resourceView[0]!=NULL);
858             }
859             if ( sys->picQuad.i_height != texDesc.Height ||
860                  sys->picQuad.i_width != texDesc.Width )
861             {
862                 /* the decoder produced different sizes than the vout, we need to
863                  * adjust the vertex */
864                 sys->picQuad.i_height = texDesc.Height;
865                 sys->picQuad.i_width = texDesc.Width;
866
867                 UpdateRects(vd, NULL, true);
868                 UpdateSize(vd);
869             }
870         }
871     }
872
873     if (subpicture) {
874         int subpicture_region_count    = 0;
875         picture_t **subpicture_regions = NULL;
876         Direct3D11MapSubpicture(vd, &subpicture_region_count, &subpicture_regions, subpicture);
877         Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
878         sys->d3dregion_count = subpicture_region_count;
879         sys->d3dregions      = subpicture_regions;
880     }
881
882     D3D11_ClearRenderTargets( &sys->d3d_dev, sys->display.pixelFormat, sys->d3drenderTargetView );
883
884     if (picture->format.mastering.max_luminance)
885     {
886         D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, GetFormatLuminance(VLC_OBJECT(vd), &picture->format) / (float)sys->display.luminance_peak);
887
888         if (sys->dxgiswapChain4)
889         {
890             DXGI_HDR_METADATA_HDR10 hdr10 = {0};
891             hdr10.GreenPrimary[0] = picture->format.mastering.primaries[0];
892             hdr10.GreenPrimary[1] = picture->format.mastering.primaries[1];
893             hdr10.BluePrimary[0]  = picture->format.mastering.primaries[2];
894             hdr10.BluePrimary[1]  = picture->format.mastering.primaries[3];
895             hdr10.RedPrimary[0]   = picture->format.mastering.primaries[4];
896             hdr10.RedPrimary[1]   = picture->format.mastering.primaries[5];
897             hdr10.WhitePoint[0] = picture->format.mastering.white_point[0];
898             hdr10.WhitePoint[1] = picture->format.mastering.white_point[1];
899             hdr10.MinMasteringLuminance = picture->format.mastering.min_luminance;
900             hdr10.MaxMasteringLuminance = picture->format.mastering.max_luminance;
901             hdr10.MaxContentLightLevel = picture->format.lighting.MaxCLL;
902             hdr10.MaxFrameAverageLightLevel = picture->format.lighting.MaxFALL;
903             IDXGISwapChain4_SetHDRMetaData(sys->dxgiswapChain4, DXGI_HDR_METADATA_TYPE_HDR10, sizeof(hdr10), &hdr10);
904         }
905     }
906
907     /* Render the quad */
908     if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader)
909         D3D11_RenderQuad(&sys->d3d_dev, &sys->picQuad, sys->stagingSys.resourceView, sys->d3drenderTargetView);
910     else {
911         picture_sys_t *p_sys = ActivePictureSys(picture);
912         D3D11_RenderQuad(&sys->d3d_dev, &sys->picQuad, p_sys->resourceView, sys->d3drenderTargetView);
913     }
914
915     if (subpicture) {
916         // draw the additional vertices
917         for (int i = 0; i < sys->d3dregion_count; ++i) {
918             if (sys->d3dregions[i])
919             {
920                 d3d_quad_t *quad = (d3d_quad_t *) sys->d3dregions[i]->p_sys;
921                 D3D11_RenderQuad(&sys->d3d_dev, quad, quad->picSys.resourceView, sys->d3drenderTargetView);
922             }
923         }
924     }
925
926     if (is_d3d11_opaque(picture->format.i_chroma))
927         d3d11_device_unlock( &sys->d3d_dev );
928
929     ID3D11DeviceContext_Flush(sys->d3d_dev.d3dcontext);
930 }
931
932 static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
933 {
934     vout_display_sys_t *sys = vd->sys;
935
936     DXGI_PRESENT_PARAMETERS presentParams;
937     memset(&presentParams, 0, sizeof(presentParams));
938     HRESULT hr = IDXGISwapChain1_Present1(sys->dxgiswapChain, 0, 0, &presentParams);
939     if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
940     {
941         /* TODO device lost */
942         msg_Dbg(vd, "SwapChain Present failed. (hr=0x%lX)", hr);
943     }
944
945     picture_Release(picture);
946     if (subpicture)
947         subpicture_Delete(subpicture);
948
949     CommonDisplay(vd);
950 }
951
952 static void Direct3D11Destroy(vout_display_t *vd)
953 {
954 #if !VLC_WINSTORE_APP
955     vout_display_sys_t *sys = vd->sys;
956     sys->hdxgi_dll = NULL;
957     D3D11_Destroy( &vd->sys->hd3d );
958 #endif
959 }
960
961 #define COLOR_RANGE_FULL   1 /* 0-255 */
962 #define COLOR_RANGE_STUDIO 0 /* 16-235 */
963
964 #define TRANSFER_FUNC_10    TRANSFER_FUNC_LINEAR
965 #define TRANSFER_FUNC_22    TRANSFER_FUNC_SRGB
966 #define TRANSFER_FUNC_2084  TRANSFER_FUNC_SMPTE_ST2084
967
968 #define COLOR_PRIMARIES_BT601  COLOR_PRIMARIES_BT601_525
969
970 static const dxgi_color_space color_spaces[] = {
971 #define DXGIMAP(AXIS, RANGE, GAMMA, SITTING, PRIMARIES) \
972     { DXGI_COLOR_SPACE_##AXIS##_##RANGE##_G##GAMMA##_##SITTING##_P##PRIMARIES, \
973       #AXIS " Rec." #PRIMARIES " gamma:" #GAMMA " range:" #RANGE, \
974       COLOR_AXIS_##AXIS, COLOR_PRIMARIES_BT##PRIMARIES, TRANSFER_FUNC_##GAMMA, \
975       COLOR_SPACE_BT##PRIMARIES, COLOR_RANGE_##RANGE},
976
977     DXGIMAP(RGB,   FULL,     22,    NONE,   709)
978     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   601)
979     DXGIMAP(YCBCR, FULL,     22,    LEFT,   601)
980     DXGIMAP(RGB,   FULL,     10,    NONE,   709)
981     DXGIMAP(RGB,   STUDIO,   22,    NONE,   709)
982     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   709)
983     DXGIMAP(YCBCR, FULL,     22,    LEFT,   709)
984     DXGIMAP(RGB,   STUDIO,   22,    NONE,  2020)
985     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,  2020)
986     DXGIMAP(YCBCR, FULL,     22,    LEFT,  2020)
987     DXGIMAP(YCBCR, STUDIO,   22, TOPLEFT,  2020)
988     DXGIMAP(RGB,   FULL,     22,    NONE,  2020)
989     DXGIMAP(RGB,   FULL,   2084,    NONE,  2020)
990     DXGIMAP(YCBCR, STUDIO, 2084,    LEFT,  2020)
991     DXGIMAP(RGB,   STUDIO, 2084,    NONE,  2020)
992     DXGIMAP(YCBCR, STUDIO, 2084, TOPLEFT,  2020)
993     /*DXGIMAP(YCBCR, FULL,     22,    NONE,  2020, 601)*/
994     {DXGI_COLOR_SPACE_RESERVED, NULL, 0, 0, 0, 0, 0},
995 #undef DXGIMAP
996 };
997
998 #ifdef HAVE_DXGI1_6_H
999 static bool canHandleConversion(const dxgi_color_space *src, const dxgi_color_space *dst)
1000 {
1001     if (src == dst)
1002         return true;
1003     if (src->primaries == COLOR_PRIMARIES_BT2020)
1004         return true; /* we can convert BT2020 to 2020 or 709 */
1005     if (dst->transfer == TRANSFER_FUNC_BT709)
1006         return true; /* we can handle anything to 709 */
1007     return false; /* let Windows do the rest */
1008 }
1009 #endif
1010
1011 static void D3D11SetColorSpace(vout_display_t *vd)
1012 {
1013     vout_display_sys_t *sys = vd->sys;
1014     HRESULT hr;
1015     int best = -1;
1016     int score, best_score = 0;
1017     UINT support;
1018     IDXGISwapChain3 *dxgiswapChain3 = NULL;
1019     sys->display.colorspace = &color_spaces[0];
1020
1021     hr = IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain3, (void **)&dxgiswapChain3);
1022     if (FAILED(hr)) {
1023         msg_Warn(vd, "could not get a IDXGISwapChain3");
1024         goto done;
1025     }
1026
1027     bool src_full_range = vd->source.b_color_range_full ||
1028                           /* the YUV->RGB conversion already output full range */
1029                           is_d3d11_opaque(vd->source.i_chroma) ||
1030                           vlc_fourcc_IsYUV(vd->source.i_chroma);
1031
1032     /* pick the best output based on color support and transfer */
1033     /* TODO support YUV output later */
1034     for (int i=0; color_spaces[i].name; ++i)
1035     {
1036         hr = IDXGISwapChain3_CheckColorSpaceSupport(dxgiswapChain3, color_spaces[i].dxgi, &support);
1037         if (SUCCEEDED(hr) && support) {
1038             msg_Dbg(vd, "supports colorspace %s", color_spaces[i].name);
1039             score = 0;
1040             if (color_spaces[i].primaries == vd->source.primaries)
1041                 score++;
1042             if (color_spaces[i].color == vd->source.space)
1043                 score += 2; /* we don't want to translate color spaces */
1044             if (color_spaces[i].transfer == vd->source.transfer ||
1045                 /* favor 2084 output for HLG source */
1046                 (color_spaces[i].transfer == TRANSFER_FUNC_SMPTE_ST2084 && vd->source.transfer == TRANSFER_FUNC_HLG))
1047                 score++;
1048             if (color_spaces[i].b_full_range == src_full_range)
1049                 score++;
1050             if (score > best_score || (score && best == -1)) {
1051                 best = i;
1052                 best_score = score;
1053             }
1054         }
1055     }
1056
1057     if (best == -1)
1058     {
1059         best = 0;
1060         msg_Warn(vd, "no matching colorspace found force %s", color_spaces[best].name);
1061     }
1062
1063 #ifdef HAVE_DXGI1_6_H
1064     IDXGIOutput *dxgiOutput = NULL;
1065
1066     if (SUCCEEDED(IDXGISwapChain_GetContainingOutput( sys->dxgiswapChain, &dxgiOutput )))
1067     {
1068         IDXGIOutput6 *dxgiOutput6 = NULL;
1069         if (SUCCEEDED(IDXGIOutput_QueryInterface( dxgiOutput, &IID_IDXGIOutput6, (void **)&dxgiOutput6 )))
1070         {
1071             DXGI_OUTPUT_DESC1 desc1;
1072             if (SUCCEEDED(IDXGIOutput6_GetDesc1( dxgiOutput6, &desc1 )))
1073             {
1074                 const dxgi_color_space *csp = NULL;
1075                 for (int i=0; color_spaces[i].name; ++i)
1076                 {
1077                     if (color_spaces[i].dxgi == desc1.ColorSpace)
1078                     {
1079                         if (!canHandleConversion(&color_spaces[best], &color_spaces[i]))
1080                             msg_Warn(vd, "Can't handle conversion to screen format %s", color_spaces[i].name);
1081                         else
1082                         {
1083                             best = i;
1084                             csp = &color_spaces[i];
1085                         }
1086                         break;
1087                     }
1088                 }
1089
1090                 msg_Dbg(vd, "Output max luminance: %.1f, colorspace %s, bits per pixel %d", desc1.MaxFullFrameLuminance, csp?csp->name:"unknown", desc1.BitsPerColor);
1091                 //sys->display.luminance_peak = desc1.MaxFullFrameLuminance;
1092             }
1093             IDXGIOutput6_Release( dxgiOutput6 );
1094         }
1095         IDXGIOutput_Release( dxgiOutput );
1096     }
1097 #endif
1098
1099     hr = IDXGISwapChain3_SetColorSpace1(dxgiswapChain3, color_spaces[best].dxgi);
1100     if (SUCCEEDED(hr))
1101     {
1102         sys->display.colorspace = &color_spaces[best];
1103         msg_Dbg(vd, "using colorspace %s", sys->display.colorspace->name);
1104     }
1105     else
1106         msg_Err(vd, "Failed to set colorspace %s. (hr=0x%lX)", sys->display.colorspace->name, hr);
1107 done:
1108     /* guestimate the display peak luminance */
1109     switch (sys->display.colorspace->transfer)
1110     {
1111     case TRANSFER_FUNC_LINEAR:
1112     case TRANSFER_FUNC_SRGB:
1113         sys->display.luminance_peak = DEFAULT_SRGB_BRIGHTNESS;
1114         break;
1115     case TRANSFER_FUNC_SMPTE_ST2084:
1116         sys->display.luminance_peak = MAX_PQ_BRIGHTNESS;
1117         break;
1118     /* there is no other output transfer on Windows */
1119     default:
1120         vlc_assert_unreachable();
1121     }
1122
1123     if (dxgiswapChain3)
1124         IDXGISwapChain3_Release(dxgiswapChain3);
1125 }
1126
1127 static const d3d_format_t *GetDirectRenderingFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1128 {
1129     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1130     if (is_d3d11_opaque(i_src_chroma))
1131         supportFlags |= D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1132     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1133 }
1134
1135 static const d3d_format_t *GetDirectDecoderFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1136 {
1137     UINT supportFlags = D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1138     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1139 }
1140
1141 static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor, bool rgb_only)
1142 {
1143     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1144     if (from_processor)
1145         supportFlags |= D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
1146     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, 0, rgb_only, bit_depth, false, supportFlags );
1147 }
1148
1149 static const d3d_format_t *GetBlendableFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1150 {
1151     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_BLENDABLE;
1152     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, false, supportFlags );
1153 }
1154
1155 static int Direct3D11Open(vout_display_t *vd)
1156 {
1157     vout_display_sys_t *sys = vd->sys;
1158     IDXGIFactory2 *dxgifactory;
1159
1160 #if !VLC_WINSTORE_APP
1161     HRESULT hr = S_OK;
1162
1163     DXGI_SWAP_CHAIN_DESC1 scd;
1164
1165     hr = D3D11_CreateDevice(vd, &sys->hd3d,
1166                             is_d3d11_opaque(vd->source.i_chroma),
1167                             &sys->d3d_dev);
1168     if (FAILED(hr)) {
1169        msg_Err(vd, "Could not Create the D3D11 device. (hr=0x%lX)", hr);
1170        return VLC_EGENERIC;
1171     }
1172
1173     IDXGIAdapter *dxgiadapter = D3D11DeviceAdapter(sys->d3d_dev.d3ddevice);
1174     if (unlikely(dxgiadapter==NULL)) {
1175        msg_Err(vd, "Could not get the DXGI Adapter");
1176        return VLC_EGENERIC;
1177     }
1178
1179     hr = IDXGIAdapter_GetParent(dxgiadapter, &IID_IDXGIFactory2, (void **)&dxgifactory);
1180     IDXGIAdapter_Release(dxgiadapter);
1181     if (FAILED(hr)) {
1182        msg_Err(vd, "Could not get the DXGI Factory. (hr=0x%lX)", hr);
1183        return VLC_EGENERIC;
1184     }
1185
1186     sys->display.pixelFormat = FindD3D11Format( sys->d3d_dev.d3ddevice, 0, true,
1187                                                 vd->source.i_chroma==VLC_CODEC_D3D11_OPAQUE_10B ? 10 : 8,
1188                                                 false, D3D11_FORMAT_SUPPORT_DISPLAY );
1189     if (unlikely(sys->display.pixelFormat == NULL))
1190         sys->display.pixelFormat = FindD3D11Format( sys->d3d_dev.d3ddevice, 0, false,
1191                                                     vd->source.i_chroma==VLC_CODEC_D3D11_OPAQUE_10B ? 10 : 8,
1192                                                     false, D3D11_FORMAT_SUPPORT_DISPLAY );
1193     if (unlikely(sys->display.pixelFormat == NULL)) {
1194         msg_Err(vd, "Could not get the SwapChain format.");
1195         return VLC_EGENERIC;
1196     }
1197
1198     FillSwapChainDesc(vd, &scd);
1199
1200     hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3d_dev.d3ddevice,
1201                                               sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1202     if (hr == DXGI_ERROR_INVALID_CALL && scd.Format == DXGI_FORMAT_R10G10B10A2_UNORM)
1203     {
1204         msg_Warn(vd, "10 bits swapchain failed, try 8 bits");
1205         scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
1206         hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3d_dev.d3ddevice,
1207                                                   sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1208     }
1209     IDXGIFactory2_Release(dxgifactory);
1210     if (FAILED(hr)) {
1211        msg_Err(vd, "Could not create the SwapChain. (hr=0x%lX)", hr);
1212        return VLC_EGENERIC;
1213     }
1214 #endif
1215
1216     IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain4, (void **)&sys->dxgiswapChain4);
1217
1218     D3D11SetColorSpace(vd);
1219
1220     video_format_t fmt;
1221     video_format_Copy(&fmt, &vd->source);
1222     int err = SetupOutputFormat(vd, &fmt);
1223     if (err != VLC_SUCCESS)
1224     {
1225         if (!is_d3d11_opaque(vd->source.i_chroma)
1226 #if !VLC_WINSTORE_APP
1227             && vd->obj.force
1228 #endif
1229                 )
1230         {
1231             const vlc_fourcc_t *list = vlc_fourcc_IsYUV(vd->source.i_chroma) ?
1232                         vlc_fourcc_GetYUVFallback(vd->source.i_chroma) :
1233                         vlc_fourcc_GetRGBFallback(vd->source.i_chroma);
1234             for (unsigned i = 0; list[i] != 0; i++) {
1235                 fmt.i_chroma = list[i];
1236                 if (fmt.i_chroma == vd->source.i_chroma)
1237                     continue;
1238                 err = SetupOutputFormat(vd, &fmt);
1239                 if (err == VLC_SUCCESS)
1240                     break;
1241             }
1242         }
1243         if (err != VLC_SUCCESS)
1244             return err;
1245     }
1246
1247     if (Direct3D11CreateGenericResources(vd)) {
1248         msg_Err(vd, "Failed to allocate resources");
1249         return VLC_EGENERIC;
1250     }
1251
1252     video_format_Clean(&vd->fmt);
1253     vd->fmt = fmt;
1254
1255     return VLC_SUCCESS;
1256 }
1257
1258 static int SetupOutputFormat(vout_display_t *vd, video_format_t *fmt)
1259 {
1260     vout_display_sys_t *sys = vd->sys;
1261
1262     // look for the requested pixel format first
1263     sys->picQuad.formatInfo = GetDirectRenderingFormat(vd, fmt->i_chroma);
1264
1265     // look for any pixel format that we can handle with enough pixels per channel
1266     const d3d_format_t *decoder_format = NULL;
1267     if ( !sys->picQuad.formatInfo )
1268     {
1269         uint8_t bits_per_channel;
1270         switch (fmt->i_chroma)
1271         {
1272         case VLC_CODEC_D3D11_OPAQUE:
1273         case VLC_CODEC_D3D11_OPAQUE_RGBA:
1274         case VLC_CODEC_D3D11_OPAQUE_BGRA:
1275             bits_per_channel = 8;
1276             break;
1277         case VLC_CODEC_D3D11_OPAQUE_10B:
1278             bits_per_channel = 10;
1279             break;
1280         default:
1281             {
1282                 const vlc_chroma_description_t *p_format = vlc_fourcc_GetChromaDescription(fmt->i_chroma);
1283                 bits_per_channel = p_format == NULL ||
1284                                    p_format->pixel_bits == 0 ? 8 : p_format->pixel_bits /
1285                                                                (p_format->plane_count==1 ? p_format->pixel_size : 1);
1286             }
1287             break;
1288         }
1289
1290         /* look for a decoder format that can be decoded but not used in shaders */
1291         if ( is_d3d11_opaque(fmt->i_chroma) )
1292             decoder_format = GetDirectDecoderFormat(vd, fmt->i_chroma);
1293         else
1294             decoder_format = sys->picQuad.formatInfo;
1295
1296         bool is_rgb = !vlc_fourcc_IsYUV(fmt->i_chroma);
1297         sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, is_rgb);
1298         if (!sys->picQuad.formatInfo && is_rgb)
1299             sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, false);
1300     }
1301
1302     // look for any pixel format that we can handle
1303     if ( !sys->picQuad.formatInfo )
1304         sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, 0, false, false);
1305
1306     if ( !sys->picQuad.formatInfo )
1307     {
1308        msg_Err(vd, "Could not get a suitable texture pixel format");
1309        return VLC_EGENERIC;
1310     }
1311
1312     fmt->i_chroma = decoder_format ? decoder_format->fourcc : sys->picQuad.formatInfo->fourcc;
1313
1314     msg_Dbg( vd, "Using pixel format %s for chroma %4.4s", sys->picQuad.formatInfo->name,
1315                  (char *)&fmt->i_chroma );
1316     DxgiFormatMask( sys->picQuad.formatInfo->formatTexture, fmt );
1317
1318     /* check the region pixel format */
1319     sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_RGBA);
1320     if (!sys->d3dregion_format)
1321         sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_BGRA);
1322
1323     if (Direct3D11CreateFormatResources(vd, fmt)) {
1324         msg_Err(vd, "Failed to allocate format resources");
1325         return VLC_EGENERIC;
1326     }
1327
1328     return VLC_SUCCESS;
1329 }
1330
1331 static void Direct3D11Close(vout_display_t *vd)
1332 {
1333     vout_display_sys_t *sys = vd->sys;
1334
1335     Direct3D11DestroyResources(vd);
1336     if (sys->dxgiswapChain4)
1337     {
1338         IDXGISwapChain_Release(sys->dxgiswapChain4);
1339         sys->dxgiswapChain4 = NULL;
1340     }
1341     if (sys->dxgiswapChain)
1342     {
1343         IDXGISwapChain_Release(sys->dxgiswapChain);
1344         sys->dxgiswapChain = NULL;
1345     }
1346
1347     D3D11_ReleaseDevice( &sys->d3d_dev );
1348
1349     msg_Dbg(vd, "Direct3D11 device adapter closed");
1350 }
1351
1352 static void UpdatePicQuadPosition(vout_display_t *vd)
1353 {
1354     vout_display_sys_t *sys = vd->sys;
1355
1356     D3D11_UpdateViewport( &sys->picQuad, &sys->sys.rect_dest_clipped, sys->display.pixelFormat );
1357
1358     SetQuadVSProjection(vd, &sys->picQuad, &vd->cfg->viewpoint);
1359
1360 #ifndef NDEBUG
1361     msg_Dbg( vd, "picQuad position (%.02f,%.02f) %.02fx%.02f",
1362              sys->picQuad.cropViewport[0].TopLeftX, sys->picQuad.cropViewport[0].TopLeftY,
1363              sys->picQuad.cropViewport[0].Width, sys->picQuad.cropViewport[0].Height );
1364 #endif
1365 }
1366
1367 static bool CanUseTextureArray(vout_display_t *vd)
1368 {
1369 #ifndef HAVE_ID3D11VIDEODECODER
1370     (void) vd;
1371     return false;
1372 #else
1373     struct wddm_version WDDM = {
1374         .wddm         = 22,
1375         .d3d_features = 19,
1376         .revision     = 162,
1377         .build        = 0,
1378     };
1379     if (D3D11CheckDriverVersion(&vd->sys->d3d_dev, GPU_MANUFACTURER_AMD, &WDDM) == VLC_SUCCESS)
1380         return true;
1381
1382     msg_Dbg(vd, "fallback to legacy shader mode for old AMD drivers");
1383     return false;
1384 #endif
1385 }
1386
1387 /* TODO : handle errors better
1388    TODO : seperate out into smaller functions like createshaders */
1389 static int Direct3D11CreateFormatResources(vout_display_t *vd, const video_format_t *fmt)
1390 {
1391     vout_display_sys_t *sys = vd->sys;
1392     HRESULT hr;
1393
1394     sys->legacy_shader = sys->d3d_dev.feature_level < D3D_FEATURE_LEVEL_10_0 || !CanUseTextureArray(vd);
1395
1396     hr = D3D11_CompilePixelShader(vd, &sys->hd3d, sys->legacy_shader, &sys->d3d_dev,
1397                                   sys->picQuad.formatInfo, &sys->display, fmt->transfer, fmt->b_color_range_full,
1398                                   sys->picQuad.d3dpixelShader, sys->picQuad.d3dsampState);
1399     if (FAILED(hr))
1400     {
1401         msg_Err(vd, "Failed to create the pixel shader. (hr=0x%lX)", hr);
1402         return VLC_EGENERIC;
1403     }
1404
1405     sys->picQuad.i_width  = fmt->i_width;
1406     sys->picQuad.i_height = fmt->i_height;
1407     if (!sys->legacy_shader && is_d3d11_opaque(fmt->i_chroma))
1408     {
1409         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x7F) & ~0x7F;
1410         sys->picQuad.i_height = (sys->picQuad.i_height + 0x7F) & ~0x7F;
1411     }
1412     else
1413     if ( sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
1414          sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
1415     {
1416         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x01) & ~0x01;
1417         sys->picQuad.i_height = (sys->picQuad.i_height + 0x01) & ~0x01;
1418     }
1419
1420     UpdateRects(vd, NULL, true);
1421
1422 #ifdef HAVE_ID3D11VIDEODECODER
1423     if (!is_d3d11_opaque(fmt->i_chroma) || sys->legacy_shader)
1424     {
1425         /* we need a staging texture */
1426         ID3D11Texture2D *textures[D3D11_MAX_SHADER_VIEW] = {0};
1427         video_format_t surface_fmt = *fmt;
1428         surface_fmt.i_width  = sys->picQuad.i_width;
1429         surface_fmt.i_height = sys->picQuad.i_height;
1430
1431         if (AllocateTextures(vd, &sys->d3d_dev, sys->picQuad.formatInfo, &surface_fmt, 1, textures))
1432         {
1433             msg_Err(vd, "Failed to allocate the staging texture");
1434             return VLC_EGENERIC;
1435         }
1436
1437         if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->picQuad.formatInfo,
1438                                      textures, 0, sys->stagingSys.resourceView))
1439         {
1440             msg_Err(vd, "Failed to allocate the staging shader view");
1441             return VLC_EGENERIC;
1442         }
1443
1444         for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
1445             sys->stagingSys.texture[plane] = textures[plane];
1446     }
1447 #endif
1448
1449     vd->info.is_slow = !is_d3d11_opaque(fmt->i_chroma) && sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_UNKNOWN;
1450     return VLC_SUCCESS;
1451 }
1452
1453 static int Direct3D11CreateGenericResources(vout_display_t *vd)
1454 {
1455     vout_display_sys_t *sys = vd->sys;
1456     HRESULT hr;
1457
1458 #if defined(HAVE_ID3D11VIDEODECODER)
1459     sys->d3d_dev.context_mutex = CreateMutexEx( NULL, NULL, 0, SYNCHRONIZE );
1460     ID3D11Device_SetPrivateData( sys->d3d_dev.d3ddevice, &GUID_CONTEXT_MUTEX,
1461                                  sizeof( sys->d3d_dev.context_mutex ), &sys->d3d_dev.context_mutex );
1462 #endif
1463
1464     ID3D11BlendState *pSpuBlendState;
1465     D3D11_BLEND_DESC spuBlendDesc = { 0 };
1466     spuBlendDesc.RenderTarget[0].BlendEnable = TRUE;
1467     spuBlendDesc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
1468     /* output colors */
1469     spuBlendDesc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD;
1470     spuBlendDesc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; /* keep source intact */
1471     spuBlendDesc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; /* RGB colors + inverse alpha (255 is full opaque) */
1472     /* output alpha  */
1473     spuBlendDesc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;
1474     spuBlendDesc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; /* keep source intact */
1475     spuBlendDesc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO; /* discard */
1476
1477     hr = ID3D11Device_CreateBlendState(sys->d3d_dev.d3ddevice, &spuBlendDesc, &pSpuBlendState);
1478     if (FAILED(hr)) {
1479        msg_Err(vd, "Could not create SPU blend state. (hr=0x%lX)", hr);
1480        return VLC_EGENERIC;
1481     }
1482     ID3D11DeviceContext_OMSetBlendState(sys->d3d_dev.d3dcontext, pSpuBlendState, NULL, 0xFFFFFFFF);
1483     ID3D11BlendState_Release(pSpuBlendState);
1484
1485     /* disable depth testing as we're only doing 2D
1486      * see https://msdn.microsoft.com/en-us/library/windows/desktop/bb205074%28v=vs.85%29.aspx
1487      * see http://rastertek.com/dx11tut11.html
1488     */
1489     D3D11_DEPTH_STENCIL_DESC stencilDesc;
1490     ZeroMemory(&stencilDesc, sizeof(stencilDesc));
1491
1492     ID3D11DepthStencilState *pDepthStencilState;
1493     hr = ID3D11Device_CreateDepthStencilState(sys->d3d_dev.d3ddevice, &stencilDesc, &pDepthStencilState );
1494     if (SUCCEEDED(hr)) {
1495         ID3D11DeviceContext_OMSetDepthStencilState(sys->d3d_dev.d3dcontext, pDepthStencilState, 0);
1496         ID3D11DepthStencilState_Release(pDepthStencilState);
1497     }
1498
1499     UpdateRects(vd, NULL, true);
1500
1501     hr = UpdateBackBuffer(vd);
1502     if (FAILED(hr)) {
1503        msg_Err(vd, "Could not update the backbuffer. (hr=0x%lX)", hr);
1504        return VLC_EGENERIC;
1505     }
1506
1507     if (sys->d3dregion_format != NULL)
1508     {
1509         hr = D3D11_CompilePixelShader(vd, &sys->hd3d, sys->legacy_shader, &sys->d3d_dev,
1510                                       sys->d3dregion_format, &sys->display, TRANSFER_FUNC_SRGB, true,
1511                                       sys->pSPUPixelShader, NULL);
1512         if (FAILED(hr))
1513         {
1514             for (size_t i=0; i<D3D11_MAX_SHADER_VIEW; i++)
1515             {
1516                 if (sys->picQuad.d3dpixelShader[i])
1517                 {
1518                     ID3D11PixelShader_Release(sys->picQuad.d3dpixelShader[i]);
1519                     sys->picQuad.d3dpixelShader[i] = NULL;
1520                 }
1521             }
1522             msg_Err(vd, "Failed to create the SPU pixel shader. (hr=0x%lX)", hr);
1523             return VLC_EGENERIC;
1524         }
1525     }
1526
1527     ID3DBlob *pVSBlob = D3D11_CompileShader(vd, &sys->hd3d, &sys->d3d_dev, globVertexShaderFlat, false);
1528     if (!pVSBlob)
1529         return VLC_EGENERIC;
1530
1531     hr = ID3D11Device_CreateVertexShader(sys->d3d_dev.d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1532                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->flatVSShader);
1533
1534     if(FAILED(hr)) {
1535       ID3D10Blob_Release(pVSBlob);
1536       msg_Err(vd, "Failed to create the flat vertex shader. (hr=0x%lX)", hr);
1537       return VLC_EGENERIC;
1538     }
1539
1540     D3D11_INPUT_ELEMENT_DESC layout[] = {
1541     { "POSITION", 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
1542     { "TEXCOORD", 0, DXGI_FORMAT_R32G32_FLOAT,    0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
1543     };
1544
1545     hr = ID3D11Device_CreateInputLayout(sys->d3d_dev.d3ddevice, layout, 2, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1546                                         ID3D10Blob_GetBufferSize(pVSBlob), &sys->pVertexLayout);
1547
1548     ID3D10Blob_Release(pVSBlob);
1549
1550     if(FAILED(hr)) {
1551       msg_Err(vd, "Failed to create the vertex input layout. (hr=0x%lX)", hr);
1552       return VLC_EGENERIC;
1553     }
1554
1555     pVSBlob = D3D11_CompileShader(vd, &sys->hd3d, &sys->d3d_dev, globVertexShaderProjection, false);
1556     if (!pVSBlob)
1557         return VLC_EGENERIC;
1558
1559     hr = ID3D11Device_CreateVertexShader(sys->d3d_dev.d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1560                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->projectionVSShader);
1561
1562     if(FAILED(hr)) {
1563       ID3D10Blob_Release(pVSBlob);
1564       msg_Err(vd, "Failed to create the projection vertex shader. (hr=0x%lX)", hr);
1565       return VLC_EGENERIC;
1566     }
1567     ID3D10Blob_Release(pVSBlob);
1568
1569     UpdatePicQuadPosition(vd);
1570
1571     msg_Dbg(vd, "Direct3D11 resources created");
1572     return VLC_SUCCESS;
1573 }
1574
1575 static void Direct3D11DestroyPool(vout_display_t *vd)
1576 {
1577     vout_display_sys_t *sys = vd->sys;
1578
1579     if (sys->sys.pool)
1580         picture_pool_Release(sys->sys.pool);
1581     sys->sys.pool = NULL;
1582 }
1583
1584 static void Direct3D11DestroyResources(vout_display_t *vd)
1585 {
1586     vout_display_sys_t *sys = vd->sys;
1587
1588     Direct3D11DestroyPool(vd);
1589
1590     D3D11_ReleaseQuad(&sys->picQuad);
1591     Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
1592     sys->d3dregion_count = 0;
1593
1594     ReleasePictureSys(&sys->stagingSys);
1595
1596     if (sys->pVertexLayout)
1597     {
1598         ID3D11InputLayout_Release(sys->pVertexLayout);
1599         sys->pVertexLayout = NULL;
1600     }
1601     if (sys->flatVSShader)
1602     {
1603         ID3D11VertexShader_Release(sys->flatVSShader);
1604         sys->flatVSShader = NULL;
1605     }
1606     if (sys->projectionVSShader)
1607     {
1608         ID3D11VertexShader_Release(sys->projectionVSShader);
1609         sys->projectionVSShader = NULL;
1610     }
1611     for (size_t i=0; i < D3D11_MAX_SHADER_VIEW; i++)
1612     {
1613         if (sys->d3drenderTargetView[i]) {
1614             ID3D11RenderTargetView_Release(sys->d3drenderTargetView[i]);
1615             sys->d3drenderTargetView[i] = NULL;
1616         }
1617         if (sys->pSPUPixelShader[i])
1618         {
1619             ID3D11PixelShader_Release(sys->pSPUPixelShader[i]);
1620             sys->pSPUPixelShader[i] = NULL;
1621         }
1622     }
1623 #if defined(HAVE_ID3D11VIDEODECODER)
1624     if( sys->d3d_dev.context_mutex != INVALID_HANDLE_VALUE )
1625     {
1626         CloseHandle( sys->d3d_dev.context_mutex );
1627         sys->d3d_dev.context_mutex = INVALID_HANDLE_VALUE;
1628     }
1629 #endif
1630
1631     msg_Dbg(vd, "Direct3D11 resources destroyed");
1632 }
1633
1634 static void Direct3D11DeleteRegions(int count, picture_t **region)
1635 {
1636     for (int i = 0; i < count; ++i) {
1637         if (region[i]) {
1638             picture_Release(region[i]);
1639         }
1640     }
1641     free(region);
1642 }
1643
1644 static void DestroyPictureQuad(picture_t *p_picture)
1645 {
1646     D3D11_ReleaseQuad( (d3d_quad_t *) p_picture->p_sys );
1647     free( p_picture );
1648 }
1649
1650 static int Direct3D11MapSubpicture(vout_display_t *vd, int *subpicture_region_count,
1651                                    picture_t ***region, subpicture_t *subpicture)
1652 {
1653     vout_display_sys_t *sys = vd->sys;
1654     D3D11_MAPPED_SUBRESOURCE mappedResource;
1655     D3D11_TEXTURE2D_DESC texDesc;
1656     HRESULT hr;
1657     int err;
1658
1659     if (sys->d3dregion_format == NULL)
1660         return VLC_EGENERIC;
1661
1662     int count = 0;
1663     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
1664         count++;
1665
1666     *region = calloc(count, sizeof(picture_t *));
1667     if (unlikely(*region==NULL))
1668         return VLC_ENOMEM;
1669     *subpicture_region_count = count;
1670
1671     int i = 0;
1672     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) {
1673         if (!r->fmt.i_visible_width || !r->fmt.i_visible_height)
1674             continue; // won't render anything, keep the cache for later
1675
1676         for (int j = 0; j < sys->d3dregion_count; j++) {
1677             picture_t *cache = sys->d3dregions[j];
1678             if (cache != NULL && ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX]) {
1679                 ID3D11Texture2D_GetDesc( ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX], &texDesc );
1680                 if (texDesc.Format == sys->d3dregion_format->formatTexture &&
1681                     texDesc.Width  == r->p_picture->format.i_width &&
1682                     texDesc.Height == r->p_picture->format.i_height) {
1683                     (*region)[i] = cache;
1684                     memset(&sys->d3dregions[j], 0, sizeof(cache)); // do not reuse this cached value
1685                     break;
1686                 }
1687             }
1688         }
1689
1690         RECT output;
1691         output.left   = r->fmt.i_x_offset;
1692         output.right  = r->fmt.i_x_offset + r->fmt.i_visible_width;
1693         output.top    = r->fmt.i_y_offset;
1694         output.bottom = r->fmt.i_y_offset + r->fmt.i_visible_height;
1695
1696         picture_t *quad_picture = (*region)[i];
1697         if (quad_picture == NULL) {
1698             d3d_quad_t *d3dquad = calloc(1, sizeof(*d3dquad));
1699             if (unlikely(d3dquad==NULL)) {
1700                 continue;
1701             }
1702             if (AllocateTextures(vd, &sys->d3d_dev, sys->d3dregion_format, &r->p_picture->format, 1, d3dquad->picSys.texture)) {
1703                 msg_Err(vd, "Failed to allocate %dx%d texture for OSD",
1704                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1705                 for (int j=0; j<D3D11_MAX_SHADER_VIEW; j++)
1706                     if (d3dquad->picSys.texture[j])
1707                         ID3D11Texture2D_Release(d3dquad->picSys.texture[j]);
1708                 free(d3dquad);
1709                 continue;
1710             }
1711
1712             if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->d3dregion_format,
1713                                          d3dquad->picSys.texture, 0,
1714                                          d3dquad->picSys.resourceView)) {
1715                 msg_Err(vd, "Failed to create %dx%d shader view for OSD",
1716                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1717                 free(d3dquad);
1718                 continue;
1719             }
1720             d3dquad->i_width    = r->fmt.i_width;
1721             d3dquad->i_height   = r->fmt.i_height;
1722
1723             d3dquad->formatInfo = sys->d3dregion_format;
1724             err = D3D11_SetupQuad( vd, &sys->d3d_dev, &r->fmt, d3dquad, &sys->display, &output,
1725                              sys->flatVSShader, sys->pVertexLayout, PROJECTION_MODE_RECTANGULAR, ORIENT_NORMAL );
1726             if (err != VLC_SUCCESS) {
1727                 msg_Err(vd, "Failed to create %dx%d quad for OSD",
1728                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1729                 free(d3dquad);
1730                 continue;
1731             }
1732             picture_resource_t picres = {
1733                 .p_sys      = (picture_sys_t *) d3dquad,
1734                 .pf_destroy = DestroyPictureQuad,
1735             };
1736             (*region)[i] = picture_NewFromResource(&r->p_picture->format, &picres);
1737             if ((*region)[i] == NULL) {
1738                 msg_Err(vd, "Failed to create %dx%d picture for OSD",
1739                         r->fmt.i_width, r->fmt.i_height);
1740                 D3D11_ReleaseQuad(d3dquad);
1741                 continue;
1742             }
1743             quad_picture = (*region)[i];
1744             for (size_t j=0; j<D3D11_MAX_SHADER_VIEW; j++)
1745             {
1746                 /* TODO use something more accurate if we have different formats */
1747                 if (sys->pSPUPixelShader[j])
1748                 {
1749                     d3dquad->d3dpixelShader[j] = sys->pSPUPixelShader[j];
1750                     ID3D11PixelShader_AddRef(d3dquad->d3dpixelShader[j]);
1751                 }
1752             }
1753         } else {
1754             D3D11_UpdateQuadPosition(vd, &sys->d3d_dev, (d3d_quad_t *) quad_picture->p_sys, &output, ORIENT_NORMAL);
1755         }
1756
1757         hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
1758         if( SUCCEEDED(hr) ) {
1759             err = CommonUpdatePicture(quad_picture, NULL, mappedResource.pData, mappedResource.RowPitch);
1760             if (err != VLC_SUCCESS) {
1761                 msg_Err(vd, "Failed to set the buffer on the SPU picture" );
1762                 ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
1763                 picture_Release(quad_picture);
1764                 if ((*region)[i] == quad_picture)
1765                     (*region)[i] = NULL;
1766                 continue;
1767             }
1768
1769             picture_CopyPixels(quad_picture, r->p_picture);
1770
1771             ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
1772         } else {
1773             msg_Err(vd, "Failed to map the SPU texture (hr=0x%lX)", hr );
1774             picture_Release(quad_picture);
1775             if ((*region)[i] == quad_picture)
1776                 (*region)[i] = NULL;
1777             continue;
1778         }
1779
1780         d3d_quad_t *quad = (d3d_quad_t *) quad_picture->p_sys;
1781
1782         RECT spuViewport;
1783         spuViewport.left   = sys->sys.rect_dest.left + (FLOAT) r->i_x * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
1784         spuViewport.top    = sys->sys.rect_dest.top  + (FLOAT) r->i_y * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
1785         spuViewport.right  = sys->sys.rect_dest.left + (FLOAT) (r->i_x + r->fmt.i_visible_width)  * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
1786         spuViewport.bottom = sys->sys.rect_dest.top  + (FLOAT) (r->i_y + r->fmt.i_visible_height) * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
1787
1788         D3D11_UpdateViewport( quad, &spuViewport, sys->display.pixelFormat );
1789
1790         D3D11_UpdateQuadOpacity(vd, &sys->d3d_dev, quad, r->i_alpha / 255.0f );
1791     }
1792     return VLC_SUCCESS;
1793 }
1794