vout: win32: direct3d11: fixup of d1e7dc1
[vlc.git] / modules / video_output / win32 / direct3d11.c
1 /*****************************************************************************
2  * direct3d11.c: Windows Direct3D11 video output module
3  *****************************************************************************
4  * Copyright (C) 2014-2015 VLC authors and VideoLAN
5  *
6  * Authors: Martell Malone <martellmalone@gmail.com>
7  *          Steve Lhomme <robux4@gmail.com>
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU Lesser General Public License as published by
11  * the Free Software Foundation; either version 2.1 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public License
20  * along with this program; if not, write to the Free Software Foundation,
21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 #ifdef HAVE_CONFIG_H
25 # include "config.h"
26 #endif
27
28 #include <vlc_common.h>
29 #include <vlc_plugin.h>
30 #include <vlc_vout_display.h>
31
32 #include <assert.h>
33 #include <math.h>
34
35 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < _WIN32_WINNT_WIN7
36 # undef _WIN32_WINNT
37 # define _WIN32_WINNT _WIN32_WINNT_WIN7
38 #endif
39
40 #define COBJMACROS
41 #include <initguid.h>
42 #include <d3d11.h>
43 #ifdef HAVE_DXGI1_6_H
44 # include <dxgi1_6.h>
45 #else
46 # include <dxgi1_5.h>
47 #endif
48
49 /* avoided until we can pass ISwapchainPanel without c++/cx mode
50 # include <windows.ui.xaml.media.dxinterop.h> */
51
52 #include "../../video_chroma/d3d11_fmt.h"
53 #include "d3d11_quad.h"
54 #include "d3d11_shaders.h"
55
56 #include "common.h"
57
58 DEFINE_GUID(GUID_SWAPCHAIN_WIDTH,  0xf1b59347, 0x1643, 0x411a, 0xad, 0x6b, 0xc7, 0x80, 0x17, 0x7a, 0x06, 0xb6);
59 DEFINE_GUID(GUID_SWAPCHAIN_HEIGHT, 0x6ea976a0, 0x9d60, 0x4bb7, 0xa5, 0xa9, 0x7d, 0xd1, 0x18, 0x7f, 0xc9, 0xbd);
60
61 static int  Open(vlc_object_t *);
62 static void Close(vlc_object_t *);
63
64 #define D3D11_HELP N_("Recommended video output for Windows 8 and later versions")
65 #define HW_BLENDING_TEXT N_("Use hardware blending support")
66 #define HW_BLENDING_LONGTEXT N_(\
67     "Try to use hardware acceleration for subtitle/OSD blending.")
68
69 vlc_module_begin ()
70     set_shortname("Direct3D11")
71     set_description(N_("Direct3D11 video output"))
72     set_help(D3D11_HELP)
73     set_category(CAT_VIDEO)
74     set_subcategory(SUBCAT_VIDEO_VOUT)
75
76     add_bool("direct3d11-hw-blending", true, HW_BLENDING_TEXT, HW_BLENDING_LONGTEXT, true)
77
78 #if VLC_WINSTORE_APP
79     add_integer("winrt-d3dcontext",    0x0, NULL, NULL, true); /* ID3D11DeviceContext* */
80     add_integer("winrt-swapchain",     0x0, NULL, NULL, true); /* IDXGISwapChain1*     */
81 #endif
82
83     set_capability("vout display", 300)
84     add_shortcut("direct3d11")
85     set_callbacks(Open, Close)
86 vlc_module_end ()
87
88 struct vout_display_sys_t
89 {
90     vout_display_sys_win32_t sys;
91
92     display_info_t           display;
93
94 #if !VLC_WINSTORE_APP
95     HINSTANCE                hdxgi_dll;        /* handle of the opened dxgi dll */
96     d3d11_handle_t           hd3d;
97 #endif
98     IDXGISwapChain1          *dxgiswapChain;   /* DXGI 1.2 swap chain */
99     IDXGISwapChain4          *dxgiswapChain4;  /* DXGI 1.5 for HDR */
100     d3d11_device_t           d3d_dev;
101     d3d_quad_t               picQuad;
102
103     picture_sys_t            stagingSys;
104
105     ID3D11RenderTargetView   *d3drenderTargetView[D3D11_MAX_SHADER_VIEW];
106
107     ID3D11InputLayout         *pVertexLayout;
108     ID3D11VertexShader        *flatVSShader;
109     ID3D11VertexShader        *projectionVSShader;
110
111     /* copy from the decoder pool into picSquad before display
112      * Uses a Texture2D with slices rather than a Texture2DArray for the decoder */
113     bool                     legacy_shader;
114
115     // SPU
116     vlc_fourcc_t             pSubpictureChromas[2];
117     ID3D11PixelShader        *pSPUPixelShader[D3D11_MAX_SHADER_VIEW];
118     const d3d_format_t       *d3dregion_format;
119     int                      d3dregion_count;
120     picture_t                **d3dregions;
121 };
122
123 #define RECTWidth(r)   (int)((r).right - (r).left)
124 #define RECTHeight(r)  (int)((r).bottom - (r).top)
125
126 static picture_pool_t *Pool(vout_display_t *, unsigned);
127
128 static void Prepare(vout_display_t *, picture_t *, subpicture_t *subpicture, vlc_tick_t);
129 static void Display(vout_display_t *, picture_t *, subpicture_t *subpicture);
130
131 static void Direct3D11Destroy(vout_display_t *);
132
133 static int  Direct3D11Open (vout_display_t *);
134 static void Direct3D11Close(vout_display_t *);
135
136 static int SetupOutputFormat(vout_display_t *, video_format_t *);
137 static int  Direct3D11CreateFormatResources (vout_display_t *, const video_format_t *);
138 static int  Direct3D11CreateGenericResources(vout_display_t *);
139 static void Direct3D11DestroyResources(vout_display_t *);
140
141 static void Direct3D11DestroyPool(vout_display_t *);
142
143 static void DestroyDisplayPoolPicture(picture_t *);
144 static void Direct3D11DeleteRegions(int, picture_t **);
145 static int Direct3D11MapSubpicture(vout_display_t *, int *, picture_t ***, subpicture_t *);
146
147 static void SetQuadVSProjection(vout_display_t *, d3d_quad_t *, const vlc_viewpoint_t *);
148 static void UpdatePicQuadPosition(vout_display_t *);
149
150 static int Control(vout_display_t *, int, va_list);
151
152 static int Direct3D11MapPoolTexture(picture_t *picture)
153 {
154     picture_sys_t *p_sys = picture->p_sys;
155     D3D11_MAPPED_SUBRESOURCE mappedResource;
156     HRESULT hr;
157
158     ID3D11Device *dev;
159     ID3D11DeviceContext_GetDevice(p_sys->context, &dev);
160     ID3D11Device_Release(dev);
161
162 #ifndef NDEBUG
163     D3D11_TEXTURE2D_DESC dsc;
164     ID3D11Texture2D_GetDesc(p_sys->texture[KNOWN_DXGI_INDEX], &dsc);
165     assert(dsc.CPUAccessFlags & D3D11_CPU_ACCESS_WRITE);
166     assert(dsc.Usage & D3D11_USAGE_DYNAMIC);
167 #endif
168
169     hr = ID3D11DeviceContext_Map(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], p_sys->slice_index, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
170     if( FAILED(hr) )
171     {
172         return VLC_EGENERIC;
173     }
174     return CommonUpdatePicture(picture, NULL, mappedResource.pData, mappedResource.RowPitch);
175 }
176
177 static void Direct3D11UnmapPoolTexture(picture_t *picture)
178 {
179     picture_sys_t *p_sys = picture->p_sys;
180     ID3D11DeviceContext_Unmap(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0);
181 }
182
183 #if !VLC_WINSTORE_APP
184 static int OpenHwnd(vout_display_t *vd)
185 {
186     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
187     if (!sys)
188         return VLC_ENOMEM;
189
190     return D3D11_Create(vd, &sys->hd3d, true);
191 }
192 #else
193 static int OpenCoreW(vout_display_t *vd)
194 {
195     IDXGISwapChain1* dxgiswapChain  = var_InheritInteger(vd, "winrt-swapchain");
196     if (!dxgiswapChain)
197         return VLC_EGENERIC;
198     ID3D11DeviceContext* d3dcontext = var_InheritInteger(vd, "winrt-d3dcontext");
199     if (!d3dcontext)
200         return VLC_EGENERIC;
201     ID3D11Device* d3ddevice = NULL;
202     ID3D11DeviceContext_GetDevice(d3dcontext, &d3ddevice);
203     if (!d3ddevice)
204         return VLC_EGENERIC;
205
206     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
207     if (!sys)
208         return VLC_ENOMEM;
209
210     sys->dxgiswapChain = dxgiswapChain;
211     sys->d3d_dev.d3ddevice     = d3ddevice;
212     sys->d3d_dev.d3dcontext    = d3dcontext;
213     sys->d3d_dev.feature_level = ID3D11Device_GetFeatureLevel(sys->d3d_dev.d3ddevice );
214     IDXGISwapChain_AddRef     (sys->dxgiswapChain);
215     ID3D11Device_AddRef       (sys->d3d_dev.d3ddevice);
216     ID3D11DeviceContext_AddRef(sys->d3d_dev.d3dcontext);
217
218     return VLC_SUCCESS;
219 }
220 #endif
221
222 #if VLC_WINSTORE_APP
223 static bool GetRect(const vout_display_sys_win32_t *p_sys, RECT *out)
224 {
225     const vout_display_sys_t *sys = (const vout_display_sys_t *)p_sys;
226     out->left   = 0;
227     out->top    = 0;
228     uint32_t i_width;
229     uint32_t i_height;
230     UINT dataSize = sizeof(i_width);
231     HRESULT hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_WIDTH, &dataSize, &i_width);
232     if (FAILED(hr)) {
233         return false;
234     }
235     dataSize = sizeof(i_height);
236     hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_HEIGHT, &dataSize, &i_height);
237     if (FAILED(hr)) {
238         return false;
239     }
240     out->right  = i_width;
241     out->bottom = i_height;
242     return true;
243 }
244 #endif
245
246 static unsigned int GetPictureWidth(const vout_display_t *vd)
247 {
248     return vd->sys->picQuad.i_width;
249 }
250
251 static unsigned int GetPictureHeight(const vout_display_t *vd)
252 {
253     return vd->sys->picQuad.i_height;
254 }
255
256 static int Open(vlc_object_t *object)
257 {
258     vout_display_t *vd = (vout_display_t *)object;
259
260 #if !VLC_WINSTORE_APP
261     /* Allow using D3D11 automatically starting from Windows 8.1 */
262     if (!vd->obj.force)
263     {
264         bool isWin81OrGreater = false;
265         HMODULE hKernel32 = GetModuleHandle(TEXT("kernel32.dll"));
266         if (likely(hKernel32 != NULL))
267             isWin81OrGreater = GetProcAddress(hKernel32, "IsProcessCritical") != NULL;
268         if (!isWin81OrGreater)
269             return VLC_EGENERIC;
270     }
271 #endif
272
273 #if !VLC_WINSTORE_APP
274     int ret = OpenHwnd(vd);
275 #else
276     int ret = OpenCoreW(vd);
277 #endif
278
279     if (ret != VLC_SUCCESS)
280         return ret;
281
282     if (CommonInit(vd))
283         goto error;
284
285 #if VLC_WINSTORE_APP
286     vd->sys->sys.pf_GetRect = GetRect;
287 #endif
288     vd->sys->sys.pf_GetPictureWidth  = GetPictureWidth;
289     vd->sys->sys.pf_GetPictureHeight = GetPictureHeight;
290
291     if (Direct3D11Open(vd)) {
292         msg_Err(vd, "Direct3D11 could not be opened");
293         goto error;
294     }
295
296 #if !VLC_WINSTORE_APP
297     EventThreadUpdateTitle(vd->sys->sys.event, VOUT_TITLE " (Direct3D11 output)");
298 #endif
299     msg_Dbg(vd, "Direct3D11 device adapter successfully initialized");
300
301     vd->info.has_double_click     = true;
302     vd->info.has_pictures_invalid = vd->info.is_slow;
303
304     if (var_InheritBool(vd, "direct3d11-hw-blending") &&
305         vd->sys->d3dregion_format != NULL)
306     {
307         vd->sys->pSubpictureChromas[0] = vd->sys->d3dregion_format->fourcc;
308         vd->sys->pSubpictureChromas[1] = 0;
309         vd->info.subpicture_chromas = vd->sys->pSubpictureChromas;
310     }
311     else
312         vd->info.subpicture_chromas = NULL;
313
314     vd->pool    = Pool;
315     vd->prepare = Prepare;
316     vd->display = Display;
317     vd->control = Control;
318
319     msg_Dbg(vd, "Direct3D11 Open Succeeded");
320
321     return VLC_SUCCESS;
322
323 error:
324     Close(object);
325     return VLC_EGENERIC;
326 }
327
328 static void Close(vlc_object_t *object)
329 {
330     vout_display_t * vd = (vout_display_t *)object;
331
332     Direct3D11Close(vd);
333     CommonClean(vd);
334     Direct3D11Destroy(vd);
335     free(vd->sys);
336 }
337
338 static picture_pool_t *Pool(vout_display_t *vd, unsigned pool_size)
339 {
340     /* compensate for extra hardware decoding pulling extra pictures from our pool */
341     pool_size += 2;
342
343     vout_display_sys_t *sys = vd->sys;
344     picture_t **pictures = NULL;
345     picture_t *picture;
346     unsigned  picture_count = 0;
347
348     if (sys->sys.pool)
349         return sys->sys.pool;
350
351     if (vd->info.is_slow)
352         pool_size = 1;
353
354     video_format_t surface_fmt = vd->fmt;
355     surface_fmt.i_width  = sys->picQuad.i_width;
356     surface_fmt.i_height = sys->picQuad.i_height;
357
358     if (D3D11_SetupQuad( vd, &sys->d3d_dev, &surface_fmt, &sys->picQuad, &sys->display, &sys->sys.rect_src_clipped,
359                    vd->fmt.projection_mode == PROJECTION_MODE_RECTANGULAR ? sys->flatVSShader : sys->projectionVSShader,
360                    sys->pVertexLayout,
361                    surface_fmt.projection_mode, vd->fmt.orientation ) != VLC_SUCCESS) {
362         msg_Err(vd, "Could not Create the main quad picture.");
363         return NULL;
364     }
365
366     if ( vd->fmt.projection_mode == PROJECTION_MODE_EQUIRECTANGULAR ||
367          vd->fmt.projection_mode == PROJECTION_MODE_CUBEMAP_LAYOUT_STANDARD )
368         SetQuadVSProjection( vd, &sys->picQuad, &vd->cfg->viewpoint );
369
370     if (!vd->info.is_slow) {
371         HRESULT           hr;
372         ID3D10Multithread *pMultithread;
373         hr = ID3D11Device_QueryInterface( sys->d3d_dev.d3ddevice, &IID_ID3D10Multithread, (void **)&pMultithread);
374         if (SUCCEEDED(hr)) {
375             ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
376             ID3D10Multithread_Release(pMultithread);
377         }
378     }
379
380     if (sys->picQuad.formatInfo->formatTexture == DXGI_FORMAT_UNKNOWN)
381         sys->sys.pool = picture_pool_NewFromFormat( &surface_fmt, pool_size );
382     else
383     {
384         ID3D11Texture2D  *textures[pool_size * D3D11_MAX_SHADER_VIEW];
385         memset(textures, 0, sizeof(textures));
386         unsigned slices = pool_size;
387         if (!CanUseVoutPool(&sys->d3d_dev, pool_size))
388             /* only provide enough for the filters, we can still do direct rendering */
389             slices = __MIN(slices, 6);
390
391         if (AllocateTextures(vd, &sys->d3d_dev, sys->picQuad.formatInfo, &surface_fmt, slices, textures))
392             goto error;
393
394         pictures = calloc(pool_size, sizeof(*pictures));
395         if (!pictures)
396             goto error;
397
398         for (picture_count = 0; picture_count < pool_size; picture_count++) {
399             picture_sys_t *picsys = calloc(1, sizeof(*picsys));
400             if (unlikely(picsys == NULL))
401                 goto error;
402
403             for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
404                 picsys->texture[plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW + plane];
405
406             picsys->slice_index = picture_count;
407             picsys->formatTexture = sys->picQuad.formatInfo->formatTexture;
408             picsys->context = sys->d3d_dev.d3dcontext;
409
410             picture_resource_t resource = {
411                 .p_sys = picsys,
412                 .pf_destroy = DestroyDisplayPoolPicture,
413             };
414
415             picture = picture_NewFromResource(&surface_fmt, &resource);
416             if (unlikely(picture == NULL)) {
417                 free(picsys);
418                 msg_Err( vd, "Failed to create picture %d in the pool.", picture_count );
419                 goto error;
420             }
421
422             pictures[picture_count] = picture;
423             /* each picture_t holds a ref to the context and release it on Destroy */
424             ID3D11DeviceContext_AddRef(picsys->context);
425         }
426
427 #ifdef HAVE_ID3D11VIDEODECODER
428         if (is_d3d11_opaque(surface_fmt.i_chroma) && !sys->legacy_shader)
429 #endif
430         {
431             for (picture_count = 0; picture_count < pool_size; picture_count++) {
432                 picture_sys_t *p_sys = pictures[picture_count]->p_sys;
433                 if (!p_sys->texture[0])
434                     continue;
435                 if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->picQuad.formatInfo,
436                                              p_sys->texture, picture_count,
437                                              p_sys->resourceView))
438                     goto error;
439             }
440         }
441
442         picture_pool_configuration_t pool_cfg = {
443             .picture       = pictures,
444             .picture_count = pool_size,
445         };
446         if (vd->info.is_slow && !is_d3d11_opaque(surface_fmt.i_chroma)) {
447             pool_cfg.lock          = Direct3D11MapPoolTexture;
448             //pool_cfg.unlock        = Direct3D11UnmapPoolTexture;
449         }
450         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
451     }
452
453 error:
454     if (sys->sys.pool == NULL) {
455         picture_pool_configuration_t pool_cfg = {
456             .picture_count = 0,
457         };
458         if (pictures) {
459             msg_Dbg(vd, "Failed to create the picture d3d11 pool");
460             for (unsigned i=0;i<picture_count; ++i)
461                 picture_Release(pictures[i]);
462             free(pictures);
463         }
464
465         /* create an empty pool to avoid crashing */
466         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
467     } else {
468         msg_Dbg(vd, "D3D11 pool succeed with %d surfaces (%dx%d) context 0x%p",
469                 pool_size, surface_fmt.i_width, surface_fmt.i_height, sys->d3d_dev.d3dcontext);
470     }
471     return sys->sys.pool;
472 }
473
474 static void DestroyDisplayPoolPicture(picture_t *picture)
475 {
476     picture_sys_t *p_sys = picture->p_sys;
477     ReleasePictureSys( p_sys );
478     free(p_sys);
479     free(picture);
480 }
481
482 #if !VLC_WINSTORE_APP
483 static void FillSwapChainDesc(vout_display_t *vd, DXGI_SWAP_CHAIN_DESC1 *out)
484 {
485     ZeroMemory(out, sizeof(*out));
486     out->BufferCount = 3;
487     out->BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
488     out->SampleDesc.Count = 1;
489     out->SampleDesc.Quality = 0;
490     out->Width = vd->source.i_visible_width;
491     out->Height = vd->source.i_visible_height;
492     out->Format = vd->sys->display.pixelFormat->formatTexture;
493     //out->Flags = 512; // DXGI_SWAP_CHAIN_FLAG_YUV_VIDEO;
494     out->SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
495 }
496 #endif
497
498 static HRESULT UpdateBackBuffer(vout_display_t *vd)
499 {
500     vout_display_sys_t *sys = vd->sys;
501     HRESULT hr;
502     ID3D11Texture2D* pBackBuffer;
503     RECT rect;
504 #if VLC_WINSTORE_APP
505     if (!GetRect(&sys->sys, &rect))
506 #endif
507         rect = sys->sys.rect_dest_clipped;
508     uint32_t i_width = RECTWidth(rect);
509     uint32_t i_height = RECTHeight(rect);
510     D3D11_TEXTURE2D_DESC dsc = { 0 };
511
512     if (sys->d3drenderTargetView[0]) {
513         ID3D11Resource *res = NULL;
514         ID3D11RenderTargetView_GetResource(sys->d3drenderTargetView[0], &res);
515         if (res)
516         {
517             ID3D11Texture2D_GetDesc((ID3D11Texture2D*) res, &dsc);
518             ID3D11Resource_Release(res);
519         }
520     }
521
522     if (dsc.Width == i_width && dsc.Height == i_height)
523         return S_OK; /* nothing changed */
524
525     for (size_t i=0; i < D3D11_MAX_SHADER_VIEW; i++)
526     {
527         if (sys->d3drenderTargetView[i]) {
528             ID3D11RenderTargetView_Release(sys->d3drenderTargetView[i]);
529             sys->d3drenderTargetView[i] = NULL;
530         }
531     }
532
533     /* TODO detect is the size is the same as the output and switch to fullscreen mode */
534     hr = IDXGISwapChain_ResizeBuffers(sys->dxgiswapChain, 0, i_width, i_height,
535         DXGI_FORMAT_UNKNOWN, 0);
536     if (FAILED(hr)) {
537        msg_Err(vd, "Failed to resize the backbuffer. (hr=0x%lX)", hr);
538        return hr;
539     }
540
541     hr = IDXGISwapChain_GetBuffer(sys->dxgiswapChain, 0, &IID_ID3D11Texture2D, (LPVOID *)&pBackBuffer);
542     if (FAILED(hr)) {
543        msg_Err(vd, "Could not get the backbuffer for the Swapchain. (hr=0x%lX)", hr);
544        return hr;
545     }
546
547     hr = D3D11_CreateRenderTargets( &sys->d3d_dev, (ID3D11Resource *)pBackBuffer,
548                                     sys->display.pixelFormat, sys->d3drenderTargetView );
549     ID3D11Texture2D_Release(pBackBuffer);
550     if (FAILED(hr)) {
551         msg_Err(vd, "Failed to create the target view. (hr=0x%lX)", hr);
552         return hr;
553     }
554
555     D3D11_ClearRenderTargets( &sys->d3d_dev, sys->display.pixelFormat, sys->d3drenderTargetView );
556
557     return S_OK;
558 }
559
560 /* rotation around the Z axis */
561 static void getZRotMatrix(float theta, FLOAT matrix[static 16])
562 {
563     float st, ct;
564
565     sincosf(theta, &st, &ct);
566
567     const FLOAT m[] = {
568     /*  x    y    z    w */
569         ct,  -st, 0.f, 0.f,
570         st,  ct,  0.f, 0.f,
571         0.f, 0.f, 1.f, 0.f,
572         0.f, 0.f, 0.f, 1.f
573     };
574
575     memcpy(matrix, m, sizeof(m));
576 }
577
578 /* rotation around the Y axis */
579 static void getYRotMatrix(float theta, FLOAT matrix[static 16])
580 {
581     float st, ct;
582
583     sincosf(theta, &st, &ct);
584
585     const FLOAT m[] = {
586     /*  x    y    z    w */
587         ct,  0.f, -st, 0.f,
588         0.f, 1.f, 0.f, 0.f,
589         st,  0.f, ct,  0.f,
590         0.f, 0.f, 0.f, 1.f
591     };
592
593     memcpy(matrix, m, sizeof(m));
594 }
595
596 /* rotation around the X axis */
597 static void getXRotMatrix(float phi, FLOAT matrix[static 16])
598 {
599     float sp, cp;
600
601     sincosf(phi, &sp, &cp);
602
603     const FLOAT m[] = {
604     /*  x    y    z    w */
605         1.f, 0.f, 0.f, 0.f,
606         0.f, cp,  sp,  0.f,
607         0.f, -sp, cp,  0.f,
608         0.f, 0.f, 0.f, 1.f
609     };
610
611     memcpy(matrix, m, sizeof(m));
612 }
613
614 static void getZoomMatrix(float zoom, FLOAT matrix[static 16]) {
615
616     const FLOAT m[] = {
617         /* x   y     z     w */
618         1.0f, 0.0f, 0.0f, 0.0f,
619         0.0f, 1.0f, 0.0f, 0.0f,
620         0.0f, 0.0f, 1.0f, 0.0f,
621         0.0f, 0.0f, zoom, 1.0f
622     };
623
624     memcpy(matrix, m, sizeof(m));
625 }
626
627 /* perspective matrix see https://www.opengl.org/sdk/docs/man2/xhtml/gluPerspective.xml */
628 static void getProjectionMatrix(float sar, float fovy, FLOAT matrix[static 16]) {
629
630     float zFar  = 1000;
631     float zNear = 0.01;
632
633     float f = 1.f / tanf(fovy / 2.f);
634
635     const FLOAT m[] = {
636         f / sar, 0.f,                   0.f,                0.f,
637         0.f,     f,                     0.f,                0.f,
638         0.f,     0.f,     (zNear + zFar) / (zNear - zFar), -1.f,
639         0.f,     0.f, (2 * zNear * zFar) / (zNear - zFar),  0.f};
640
641      memcpy(matrix, m, sizeof(m));
642 }
643
644 static float UpdateFOVy(float f_fovx, float f_sar)
645 {
646     return 2 * atanf(tanf(f_fovx / 2) / f_sar);
647 }
648
649 static float UpdateZ(float f_fovx, float f_fovy)
650 {
651     /* Do trigonometry to calculate the minimal z value
652      * that will allow us to zoom out without seeing the outside of the
653      * sphere (black borders). */
654     float tan_fovx_2 = tanf(f_fovx / 2);
655     float tan_fovy_2 = tanf(f_fovy / 2);
656     float z_min = - SPHERE_RADIUS / sinf(atanf(sqrtf(
657                     tan_fovx_2 * tan_fovx_2 + tan_fovy_2 * tan_fovy_2)));
658
659     /* The FOV value above which z is dynamically calculated. */
660     const float z_thresh = 90.f;
661
662     float f_z;
663     if (f_fovx <= z_thresh * M_PI / 180)
664         f_z = 0;
665     else
666     {
667         float f = z_min / ((FIELD_OF_VIEW_DEGREES_MAX - z_thresh) * M_PI / 180);
668         f_z = f * f_fovx - f * z_thresh * M_PI / 180;
669         if (f_z < z_min)
670             f_z = z_min;
671     }
672     return f_z;
673 }
674
675 static void SetQuadVSProjection(vout_display_t *vd, d3d_quad_t *quad, const vlc_viewpoint_t *p_vp)
676 {
677     if (!quad->pVertexShaderConstants)
678         return;
679
680 #define RAD(d) ((float) ((d) * M_PI / 180.f))
681     float f_fovx = RAD(p_vp->fov);
682     if ( f_fovx > FIELD_OF_VIEW_DEGREES_MAX * M_PI / 180 + 0.001f ||
683          f_fovx < -0.001f )
684         return;
685
686     float f_sar = (float) vd->cfg->display.width / vd->cfg->display.height;
687     float f_teta = RAD(p_vp->yaw) - (float) M_PI_2;
688     float f_phi  = RAD(p_vp->pitch);
689     float f_roll = RAD(p_vp->roll);
690     float f_fovy = UpdateFOVy(f_fovx, f_sar);
691     float f_z = UpdateZ(f_fovx, f_fovy);
692
693     vout_display_sys_t *sys = vd->sys;
694     HRESULT hr;
695     D3D11_MAPPED_SUBRESOURCE mapped;
696     hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped);
697     if (SUCCEEDED(hr)) {
698         VS_PROJECTION_CONST *dst_data = mapped.pData;
699         getXRotMatrix(f_phi, dst_data->RotX);
700         getYRotMatrix(f_teta,   dst_data->RotY);
701         getZRotMatrix(f_roll,  dst_data->RotZ);
702         getZoomMatrix(SPHERE_RADIUS * f_z, dst_data->View);
703         getProjectionMatrix(f_sar, f_fovy, dst_data->Projection);
704     }
705     ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0);
706 #undef RAD
707 }
708
709 static void UpdateSize(vout_display_t *vd)
710 {
711     vout_display_sys_t *sys = vd->sys;
712     msg_Dbg(vd, "Detected size change %dx%d", RECTWidth(sys->sys.rect_dest_clipped),
713             RECTHeight(sys->sys.rect_dest_clipped));
714
715     UpdateBackBuffer(vd);
716
717     d3d11_device_lock( &sys->d3d_dev );
718
719     UpdatePicQuadPosition(vd);
720
721     D3D11_UpdateQuadPosition(vd, &sys->d3d_dev, &sys->picQuad, &sys->sys.rect_src_clipped,
722                              vd->fmt.orientation);
723
724     d3d11_device_unlock( &sys->d3d_dev );
725 }
726
727 static inline bool RectEquals(const RECT *r1, const RECT *r2)
728 {
729     return r1->bottom == r2->bottom && r1->top == r2->top &&
730            r1->left == r2->left && r1->right == r2->right;
731 }
732
733 static int Control(vout_display_t *vd, int query, va_list args)
734 {
735     vout_display_sys_t *sys = vd->sys;
736     RECT before_src_clipped  = sys->sys.rect_src_clipped;
737     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
738     RECT before_dest         = sys->sys.rect_dest;
739
740     int res = CommonControl( vd, query, args );
741
742     if (query == VOUT_DISPLAY_CHANGE_VIEWPOINT)
743     {
744         const vout_display_cfg_t *cfg = va_arg(args, const vout_display_cfg_t*);
745         if ( sys->picQuad.pVertexShaderConstants )
746         {
747             SetQuadVSProjection( vd, &sys->picQuad, &cfg->viewpoint );
748             res = VLC_SUCCESS;
749         }
750     }
751
752     if (!RectEquals(&before_src_clipped,  &sys->sys.rect_src_clipped) ||
753         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
754         !RectEquals(&before_dest,         &sys->sys.rect_dest) )
755     {
756         UpdateSize(vd);
757     }
758
759     return res;
760 }
761
762 static void Manage(vout_display_t *vd)
763 {
764     vout_display_sys_t *sys = vd->sys;
765     RECT before_src_clipped  = sys->sys.rect_src_clipped;
766     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
767     RECT before_dest         = sys->sys.rect_dest;
768
769     CommonManage(vd);
770
771     if (!RectEquals(&before_src_clipped, &sys->sys.rect_src_clipped) ||
772         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
773         !RectEquals(&before_dest, &sys->sys.rect_dest))
774     {
775         UpdateSize(vd);
776     }
777 }
778
779 static void Prepare(vout_display_t *vd, picture_t *picture,
780                     subpicture_t *subpicture, vlc_tick_t date)
781 {
782     Manage(vd);
783     VLC_UNUSED(date);
784     vout_display_sys_t *sys = vd->sys;
785
786     if (sys->picQuad.formatInfo->formatTexture == DXGI_FORMAT_UNKNOWN)
787     {
788         D3D11_MAPPED_SUBRESOURCE mappedResource;
789         D3D11_TEXTURE2D_DESC texDesc;
790         int i;
791         HRESULT hr;
792         plane_t planes[PICTURE_PLANE_MAX];
793
794         bool b_mapped = true;
795         for (i = 0; i < picture->i_planes; i++) {
796             hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i],
797                                          0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
798             if( unlikely(FAILED(hr)) )
799             {
800                 while (i-- > 0)
801                     ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i], 0);
802                 b_mapped = false;
803                 break;
804             }
805             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[i], &texDesc);
806             planes[i].i_lines = texDesc.Height;
807             planes[i].i_pitch = mappedResource.RowPitch;
808             planes[i].p_pixels = mappedResource.pData;
809
810             planes[i].i_visible_lines = picture->p[i].i_visible_lines;
811             planes[i].i_visible_pitch = picture->p[i].i_visible_pitch;
812         }
813
814         if (b_mapped)
815         {
816             for (i = 0; i < picture->i_planes; i++)
817                 plane_CopyPixels(&planes[i], &picture->p[i]);
818
819             for (i = 0; i < picture->i_planes; i++)
820                 ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, sys->stagingSys.resource[i], 0);
821         }
822     }
823     else
824     {
825         picture_sys_t *p_sys = ActivePictureSys(picture);
826
827         if (is_d3d11_opaque(picture->format.i_chroma))
828             d3d11_device_lock( &sys->d3d_dev );
829
830         if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader) {
831             D3D11_TEXTURE2D_DESC srcDesc,texDesc;
832             if (!is_d3d11_opaque(picture->format.i_chroma))
833                 Direct3D11UnmapPoolTexture(picture);
834             ID3D11Texture2D_GetDesc(p_sys->texture[KNOWN_DXGI_INDEX], &srcDesc);
835             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[0], &texDesc);
836             D3D11_BOX box = {
837                 .top = 0,
838                 .bottom = __MIN(srcDesc.Height, texDesc.Height),
839                 .left = 0,
840                 .right = __MIN(srcDesc.Width, texDesc.Width),
841                 .back = 1,
842             };
843             ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev.d3dcontext,
844                                                       sys->stagingSys.resource[KNOWN_DXGI_INDEX],
845                                                       0, 0, 0, 0,
846                                                       p_sys->resource[KNOWN_DXGI_INDEX],
847                                                       p_sys->slice_index, &box);
848         }
849         else
850         {
851             D3D11_TEXTURE2D_DESC texDesc;
852             ID3D11Texture2D_GetDesc(p_sys->texture[0], &texDesc);
853             if (texDesc.BindFlags & D3D11_BIND_SHADER_RESOURCE)
854             {
855                 /* for performance reason we don't want to allocate this during
856                  * display, do it preferrably when creating the texture */
857                 assert(p_sys->resourceView[0]!=NULL);
858             }
859             if ( sys->picQuad.i_height != texDesc.Height ||
860                  sys->picQuad.i_width != texDesc.Width )
861             {
862                 /* the decoder produced different sizes than the vout, we need to
863                  * adjust the vertex */
864                 sys->picQuad.i_height = texDesc.Height;
865                 sys->picQuad.i_width = texDesc.Width;
866
867                 UpdateRects(vd, NULL, true);
868                 UpdateSize(vd);
869             }
870         }
871     }
872
873     if (subpicture) {
874         int subpicture_region_count    = 0;
875         picture_t **subpicture_regions = NULL;
876         Direct3D11MapSubpicture(vd, &subpicture_region_count, &subpicture_regions, subpicture);
877         Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
878         sys->d3dregion_count = subpicture_region_count;
879         sys->d3dregions      = subpicture_regions;
880     }
881
882     D3D11_ClearRenderTargets( &sys->d3d_dev, sys->display.pixelFormat, sys->d3drenderTargetView );
883
884     if (picture->format.mastering.max_luminance)
885     {
886         D3D11_UpdateQuadLuminanceScale(vd, &sys->d3d_dev, &sys->picQuad, GetFormatLuminance(VLC_OBJECT(vd), &picture->format) / (float)sys->display.luminance_peak);
887
888         if (sys->dxgiswapChain4)
889         {
890             DXGI_HDR_METADATA_HDR10 hdr10 = {0};
891             hdr10.GreenPrimary[0] = picture->format.mastering.primaries[0];
892             hdr10.GreenPrimary[1] = picture->format.mastering.primaries[1];
893             hdr10.BluePrimary[0]  = picture->format.mastering.primaries[2];
894             hdr10.BluePrimary[1]  = picture->format.mastering.primaries[3];
895             hdr10.RedPrimary[0]   = picture->format.mastering.primaries[4];
896             hdr10.RedPrimary[1]   = picture->format.mastering.primaries[5];
897             hdr10.WhitePoint[0] = picture->format.mastering.white_point[0];
898             hdr10.WhitePoint[1] = picture->format.mastering.white_point[1];
899             hdr10.MinMasteringLuminance = picture->format.mastering.min_luminance;
900             hdr10.MaxMasteringLuminance = picture->format.mastering.max_luminance;
901             hdr10.MaxContentLightLevel = picture->format.lighting.MaxCLL;
902             hdr10.MaxFrameAverageLightLevel = picture->format.lighting.MaxFALL;
903             IDXGISwapChain4_SetHDRMetaData(sys->dxgiswapChain4, DXGI_HDR_METADATA_TYPE_HDR10, sizeof(hdr10), &hdr10);
904         }
905     }
906
907     /* Render the quad */
908     if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader)
909         D3D11_RenderQuad(&sys->d3d_dev, &sys->picQuad, sys->stagingSys.resourceView, sys->d3drenderTargetView);
910     else {
911         picture_sys_t *p_sys = ActivePictureSys(picture);
912         D3D11_RenderQuad(&sys->d3d_dev, &sys->picQuad, p_sys->resourceView, sys->d3drenderTargetView);
913     }
914
915     if (subpicture) {
916         // draw the additional vertices
917         for (int i = 0; i < sys->d3dregion_count; ++i) {
918             if (sys->d3dregions[i])
919             {
920                 d3d_quad_t *quad = (d3d_quad_t *) sys->d3dregions[i]->p_sys;
921                 D3D11_RenderQuad(&sys->d3d_dev, quad, quad->picSys.resourceView, sys->d3drenderTargetView);
922             }
923         }
924     }
925
926     if (is_d3d11_opaque(picture->format.i_chroma))
927         d3d11_device_unlock( &sys->d3d_dev );
928
929     ID3D11DeviceContext_Flush(sys->d3d_dev.d3dcontext);
930 }
931
932 static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
933 {
934     vout_display_sys_t *sys = vd->sys;
935
936     DXGI_PRESENT_PARAMETERS presentParams;
937     memset(&presentParams, 0, sizeof(presentParams));
938     HRESULT hr = IDXGISwapChain1_Present1(sys->dxgiswapChain, 0, 0, &presentParams);
939     if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
940     {
941         /* TODO device lost */
942         msg_Dbg(vd, "SwapChain Present failed. (hr=0x%lX)", hr);
943     }
944
945     picture_Release(picture);
946     if (subpicture)
947         subpicture_Delete(subpicture);
948
949     CommonDisplay(vd);
950 }
951
952 static void Direct3D11Destroy(vout_display_t *vd)
953 {
954 #if !VLC_WINSTORE_APP
955     vout_display_sys_t *sys = vd->sys;
956     sys->hdxgi_dll = NULL;
957     D3D11_Destroy( &vd->sys->hd3d );
958 #endif
959 }
960
961 #define COLOR_RANGE_FULL   1 /* 0-255 */
962 #define COLOR_RANGE_STUDIO 0 /* 16-235 */
963
964 #define TRANSFER_FUNC_10    TRANSFER_FUNC_LINEAR
965 #define TRANSFER_FUNC_22    TRANSFER_FUNC_SRGB
966 #define TRANSFER_FUNC_2084  TRANSFER_FUNC_SMPTE_ST2084
967
968 #define COLOR_PRIMARIES_BT601  COLOR_PRIMARIES_BT601_525
969
970 static const dxgi_color_space color_spaces[] = {
971 #define DXGIMAP(AXIS, RANGE, GAMMA, SITTING, PRIMARIES) \
972     { DXGI_COLOR_SPACE_##AXIS##_##RANGE##_G##GAMMA##_##SITTING##_P##PRIMARIES, \
973       #AXIS " Rec." #PRIMARIES " gamma:" #GAMMA " range:" #RANGE, \
974       COLOR_AXIS_##AXIS, COLOR_PRIMARIES_BT##PRIMARIES, TRANSFER_FUNC_##GAMMA, \
975       COLOR_SPACE_BT##PRIMARIES, COLOR_RANGE_##RANGE},
976
977     DXGIMAP(RGB,   FULL,     22,    NONE,   709)
978     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   601)
979     DXGIMAP(YCBCR, FULL,     22,    LEFT,   601)
980     DXGIMAP(RGB,   FULL,     10,    NONE,   709)
981     DXGIMAP(RGB,   STUDIO,   22,    NONE,   709)
982     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   709)
983     DXGIMAP(YCBCR, FULL,     22,    LEFT,   709)
984     DXGIMAP(RGB,   STUDIO,   22,    NONE,  2020)
985     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,  2020)
986     DXGIMAP(YCBCR, FULL,     22,    LEFT,  2020)
987     DXGIMAP(YCBCR, STUDIO,   22, TOPLEFT,  2020)
988     DXGIMAP(RGB,   FULL,     22,    NONE,  2020)
989     DXGIMAP(RGB,   FULL,   2084,    NONE,  2020)
990     DXGIMAP(YCBCR, STUDIO, 2084,    LEFT,  2020)
991     DXGIMAP(RGB,   STUDIO, 2084,    NONE,  2020)
992     DXGIMAP(YCBCR, STUDIO, 2084, TOPLEFT,  2020)
993     /*DXGIMAP(YCBCR, FULL,     22,    NONE,  2020, 601)*/
994     {DXGI_COLOR_SPACE_RESERVED, NULL, 0, 0, 0, 0, 0},
995 #undef DXGIMAP
996 };
997
998 #ifdef HAVE_DXGI1_6_H
999 static bool canHandleConversion(const dxgi_color_space *src, const dxgi_color_space *dst)
1000 {
1001     if (src == dst)
1002         return true;
1003     if (src->primaries == COLOR_PRIMARIES_BT2020)
1004         return true; /* we can convert BT2020 to 2020 or 709 */
1005     if (dst->transfer == TRANSFER_FUNC_BT709)
1006         return true; /* we can handle anything to 709 */
1007     return false; /* let Windows do the rest */
1008 }
1009 #endif
1010
1011 static void D3D11SetColorSpace(vout_display_t *vd)
1012 {
1013     vout_display_sys_t *sys = vd->sys;
1014     HRESULT hr;
1015     int best = -1;
1016     int score, best_score = 0;
1017     UINT support;
1018     IDXGISwapChain3 *dxgiswapChain3 = NULL;
1019     sys->display.colorspace = &color_spaces[0];
1020
1021     hr = IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain3, (void **)&dxgiswapChain3);
1022     if (FAILED(hr)) {
1023         msg_Warn(vd, "could not get a IDXGISwapChain3");
1024         goto done;
1025     }
1026
1027     bool src_full_range = vd->source.b_color_range_full ||
1028                           /* the YUV->RGB conversion already output full range */
1029                           is_d3d11_opaque(vd->source.i_chroma) ||
1030                           vlc_fourcc_IsYUV(vd->source.i_chroma);
1031
1032     /* pick the best output based on color support and transfer */
1033     /* TODO support YUV output later */
1034     for (int i=0; color_spaces[i].name; ++i)
1035     {
1036         hr = IDXGISwapChain3_CheckColorSpaceSupport(dxgiswapChain3, color_spaces[i].dxgi, &support);
1037         if (SUCCEEDED(hr) && support) {
1038             msg_Dbg(vd, "supports colorspace %s", color_spaces[i].name);
1039             score = 0;
1040             if (color_spaces[i].primaries == vd->source.primaries)
1041                 score++;
1042             if (color_spaces[i].color == vd->source.space)
1043                 score += 2; /* we don't want to translate color spaces */
1044             if (color_spaces[i].transfer == vd->source.transfer ||
1045                 /* favor 2084 output for HLG source */
1046                 (color_spaces[i].transfer == TRANSFER_FUNC_SMPTE_ST2084 && vd->source.transfer == TRANSFER_FUNC_HLG))
1047                 score++;
1048             if (color_spaces[i].b_full_range == src_full_range)
1049                 score++;
1050             if (score > best_score || (score && best == -1)) {
1051                 best = i;
1052                 best_score = score;
1053             }
1054         }
1055     }
1056
1057     if (best == -1)
1058     {
1059         best = 0;
1060         msg_Warn(vd, "no matching colorspace found force %s", color_spaces[best].name);
1061     }
1062
1063 #ifdef HAVE_DXGI1_6_H
1064     IDXGIOutput *dxgiOutput = NULL;
1065
1066     if (SUCCEEDED(IDXGISwapChain_GetContainingOutput( sys->dxgiswapChain, &dxgiOutput )))
1067     {
1068         IDXGIOutput6 *dxgiOutput6 = NULL;
1069         if (SUCCEEDED(IDXGIOutput_QueryInterface( dxgiOutput, &IID_IDXGIOutput6, (void **)&dxgiOutput6 )))
1070         {
1071             DXGI_OUTPUT_DESC1 desc1;
1072             if (SUCCEEDED(IDXGIOutput6_GetDesc1( dxgiOutput6, &desc1 )))
1073             {
1074                 const dxgi_color_space *csp = NULL;
1075                 for (int i=0; color_spaces[i].name; ++i)
1076                 {
1077                     if (color_spaces[i].dxgi == desc1.ColorSpace)
1078                     {
1079                         if (!canHandleConversion(&color_spaces[best], &color_spaces[i]))
1080                             msg_Warn(vd, "Can't handle conversion to screen format %s", color_spaces[i].name);
1081                         else
1082                         {
1083                             best = i;
1084                             csp = &color_spaces[i];
1085                         }
1086                         break;
1087                     }
1088                 }
1089
1090                 msg_Dbg(vd, "Output max luminance: %.1f, colorspace %s, bits per pixel %d", desc1.MaxFullFrameLuminance, csp?csp->name:"unknown", desc1.BitsPerColor);
1091                 //sys->display.luminance_peak = desc1.MaxFullFrameLuminance;
1092             }
1093             IDXGIOutput6_Release( dxgiOutput6 );
1094         }
1095         IDXGIOutput_Release( dxgiOutput );
1096     }
1097 #endif
1098
1099     hr = IDXGISwapChain3_SetColorSpace1(dxgiswapChain3, color_spaces[best].dxgi);
1100     if (SUCCEEDED(hr))
1101     {
1102         sys->display.colorspace = &color_spaces[best];
1103         msg_Dbg(vd, "using colorspace %s", sys->display.colorspace->name);
1104     }
1105     else
1106         msg_Err(vd, "Failed to set colorspace %s. (hr=0x%lX)", sys->display.colorspace->name, hr);
1107 done:
1108     /* guestimate the display peak luminance */
1109     switch (sys->display.colorspace->transfer)
1110     {
1111     case TRANSFER_FUNC_LINEAR:
1112     case TRANSFER_FUNC_SRGB:
1113         sys->display.luminance_peak = DEFAULT_SRGB_BRIGHTNESS;
1114         break;
1115     case TRANSFER_FUNC_SMPTE_ST2084:
1116         sys->display.luminance_peak = MAX_PQ_BRIGHTNESS;
1117         break;
1118     /* there is no other output transfer on Windows */
1119     default:
1120         vlc_assert_unreachable();
1121     }
1122
1123     if (dxgiswapChain3)
1124         IDXGISwapChain3_Release(dxgiswapChain3);
1125 }
1126
1127 static const d3d_format_t *GetDirectRenderingFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1128 {
1129     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1130     if (is_d3d11_opaque(i_src_chroma))
1131         supportFlags |= D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1132     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1133 }
1134
1135 static const d3d_format_t *GetDirectDecoderFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1136 {
1137     UINT supportFlags = D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1138     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1139 }
1140
1141 static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor, bool rgb_only)
1142 {
1143     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1144     if (from_processor)
1145         supportFlags |= D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
1146     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, 0, rgb_only, bit_depth, false, supportFlags );
1147 }
1148
1149 static const d3d_format_t *GetBlendableFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1150 {
1151     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_BLENDABLE;
1152     return FindD3D11Format( vd->sys->d3d_dev.d3ddevice, i_src_chroma, false, 0, false, supportFlags );
1153 }
1154
1155 static int Direct3D11Open(vout_display_t *vd)
1156 {
1157     vout_display_sys_t *sys = vd->sys;
1158     IDXGIFactory2 *dxgifactory;
1159
1160 #if !VLC_WINSTORE_APP
1161     HRESULT hr = S_OK;
1162
1163     DXGI_SWAP_CHAIN_DESC1 scd;
1164
1165     hr = D3D11_CreateDevice(vd, &sys->hd3d,
1166                             is_d3d11_opaque(vd->source.i_chroma),
1167                             &sys->d3d_dev);
1168     if (FAILED(hr)) {
1169        msg_Err(vd, "Could not Create the D3D11 device. (hr=0x%lX)", hr);
1170        return VLC_EGENERIC;
1171     }
1172
1173     IDXGIAdapter *dxgiadapter = D3D11DeviceAdapter(sys->d3d_dev.d3ddevice);
1174     if (unlikely(dxgiadapter==NULL)) {
1175        msg_Err(vd, "Could not get the DXGI Adapter");
1176        return VLC_EGENERIC;
1177     }
1178
1179     hr = IDXGIAdapter_GetParent(dxgiadapter, &IID_IDXGIFactory2, (void **)&dxgifactory);
1180     IDXGIAdapter_Release(dxgiadapter);
1181     if (FAILED(hr)) {
1182        msg_Err(vd, "Could not get the DXGI Factory. (hr=0x%lX)", hr);
1183        return VLC_EGENERIC;
1184     }
1185
1186     sys->display.pixelFormat = FindD3D11Format( sys->d3d_dev.d3ddevice, 0, true,
1187                                                 vd->source.i_chroma==VLC_CODEC_D3D11_OPAQUE_10B ? 10 : 8,
1188                                                 false, D3D11_FORMAT_SUPPORT_DISPLAY );
1189     if (unlikely(sys->display.pixelFormat == NULL))
1190         sys->display.pixelFormat = FindD3D11Format( sys->d3d_dev.d3ddevice, 0, false,
1191                                                     vd->source.i_chroma==VLC_CODEC_D3D11_OPAQUE_10B ? 10 : 8,
1192                                                     false, D3D11_FORMAT_SUPPORT_DISPLAY );
1193     if (unlikely(sys->display.pixelFormat == NULL)) {
1194         msg_Err(vd, "Could not get the SwapChain format.");
1195         return VLC_EGENERIC;
1196     }
1197
1198     FillSwapChainDesc(vd, &scd);
1199
1200     hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3d_dev.d3ddevice,
1201                                               sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1202     if (hr == DXGI_ERROR_INVALID_CALL && scd.Format == DXGI_FORMAT_R10G10B10A2_UNORM)
1203     {
1204         msg_Warn(vd, "10 bits swapchain failed, try 8 bits");
1205         scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
1206         hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3d_dev.d3ddevice,
1207                                                   sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1208     }
1209     IDXGIFactory2_Release(dxgifactory);
1210     if (FAILED(hr)) {
1211        msg_Err(vd, "Could not create the SwapChain. (hr=0x%lX)", hr);
1212        return VLC_EGENERIC;
1213     }
1214 #endif
1215
1216     IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain4, (void **)&sys->dxgiswapChain4);
1217
1218     D3D11SetColorSpace(vd);
1219
1220     video_format_t fmt;
1221     video_format_Copy(&fmt, &vd->source);
1222     int err = SetupOutputFormat(vd, &fmt);
1223     if (err != VLC_SUCCESS)
1224     {
1225         if (!is_d3d11_opaque(vd->source.i_chroma)
1226 #if !VLC_WINSTORE_APP
1227             && vd->obj.force
1228 #endif
1229                 )
1230         {
1231             const vlc_fourcc_t *list = vlc_fourcc_IsYUV(vd->source.i_chroma) ?
1232                         vlc_fourcc_GetYUVFallback(vd->source.i_chroma) :
1233                         vlc_fourcc_GetRGBFallback(vd->source.i_chroma);
1234             for (unsigned i = 0; list[i] != 0; i++) {
1235                 fmt.i_chroma = list[i];
1236                 if (fmt.i_chroma == vd->source.i_chroma)
1237                     continue;
1238                 err = SetupOutputFormat(vd, &fmt);
1239                 if (err == VLC_SUCCESS)
1240                     break;
1241             }
1242         }
1243         if (err != VLC_SUCCESS)
1244             return err;
1245     }
1246
1247     if (Direct3D11CreateGenericResources(vd)) {
1248         msg_Err(vd, "Failed to allocate resources");
1249         Direct3D11DestroyResources(vd);
1250         return VLC_EGENERIC;
1251     }
1252
1253     video_format_Clean(&vd->fmt);
1254     vd->fmt = fmt;
1255
1256     return VLC_SUCCESS;
1257 }
1258
1259 static int SetupOutputFormat(vout_display_t *vd, video_format_t *fmt)
1260 {
1261     vout_display_sys_t *sys = vd->sys;
1262
1263     // look for the requested pixel format first
1264     sys->picQuad.formatInfo = GetDirectRenderingFormat(vd, fmt->i_chroma);
1265
1266     // look for any pixel format that we can handle with enough pixels per channel
1267     const d3d_format_t *decoder_format = NULL;
1268     if ( !sys->picQuad.formatInfo )
1269     {
1270         uint8_t bits_per_channel;
1271         switch (fmt->i_chroma)
1272         {
1273         case VLC_CODEC_D3D11_OPAQUE:
1274         case VLC_CODEC_D3D11_OPAQUE_RGBA:
1275         case VLC_CODEC_D3D11_OPAQUE_BGRA:
1276             bits_per_channel = 8;
1277             break;
1278         case VLC_CODEC_D3D11_OPAQUE_10B:
1279             bits_per_channel = 10;
1280             break;
1281         default:
1282             {
1283                 const vlc_chroma_description_t *p_format = vlc_fourcc_GetChromaDescription(fmt->i_chroma);
1284                 bits_per_channel = p_format == NULL ||
1285                                    p_format->pixel_bits == 0 ? 8 : p_format->pixel_bits /
1286                                                                (p_format->plane_count==1 ? p_format->pixel_size : 1);
1287             }
1288             break;
1289         }
1290
1291         /* look for a decoder format that can be decoded but not used in shaders */
1292         if ( is_d3d11_opaque(fmt->i_chroma) )
1293             decoder_format = GetDirectDecoderFormat(vd, fmt->i_chroma);
1294         else
1295             decoder_format = sys->picQuad.formatInfo;
1296
1297         bool is_rgb = !vlc_fourcc_IsYUV(fmt->i_chroma);
1298         sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, is_rgb);
1299         if (!sys->picQuad.formatInfo && is_rgb)
1300             sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL, false);
1301     }
1302
1303     // look for any pixel format that we can handle
1304     if ( !sys->picQuad.formatInfo )
1305         sys->picQuad.formatInfo = GetDisplayFormatByDepth(vd, 0, false, false);
1306
1307     if ( !sys->picQuad.formatInfo )
1308     {
1309        msg_Err(vd, "Could not get a suitable texture pixel format");
1310        return VLC_EGENERIC;
1311     }
1312
1313     fmt->i_chroma = decoder_format ? decoder_format->fourcc : sys->picQuad.formatInfo->fourcc;
1314
1315     msg_Dbg( vd, "Using pixel format %s for chroma %4.4s", sys->picQuad.formatInfo->name,
1316                  (char *)&fmt->i_chroma );
1317     DxgiFormatMask( sys->picQuad.formatInfo->formatTexture, fmt );
1318
1319     /* check the region pixel format */
1320     sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_RGBA);
1321     if (!sys->d3dregion_format)
1322         sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_BGRA);
1323
1324     if (Direct3D11CreateFormatResources(vd, fmt)) {
1325         msg_Err(vd, "Failed to allocate format resources");
1326         Direct3D11DestroyResources(vd);
1327         return VLC_EGENERIC;
1328     }
1329
1330     return VLC_SUCCESS;
1331 }
1332
1333 static void Direct3D11Close(vout_display_t *vd)
1334 {
1335     vout_display_sys_t *sys = vd->sys;
1336
1337     Direct3D11DestroyResources(vd);
1338     if (sys->dxgiswapChain4)
1339     {
1340         IDXGISwapChain_Release(sys->dxgiswapChain4);
1341         sys->dxgiswapChain4 = NULL;
1342     }
1343     if (sys->dxgiswapChain)
1344     {
1345         IDXGISwapChain_Release(sys->dxgiswapChain);
1346         sys->dxgiswapChain = NULL;
1347     }
1348
1349     D3D11_ReleaseDevice( &sys->d3d_dev );
1350
1351     msg_Dbg(vd, "Direct3D11 device adapter closed");
1352 }
1353
1354 static void UpdatePicQuadPosition(vout_display_t *vd)
1355 {
1356     vout_display_sys_t *sys = vd->sys;
1357
1358     D3D11_UpdateViewport( &sys->picQuad, &sys->sys.rect_dest_clipped, sys->display.pixelFormat );
1359
1360     SetQuadVSProjection(vd, &sys->picQuad, &vd->cfg->viewpoint);
1361
1362 #ifndef NDEBUG
1363     msg_Dbg( vd, "picQuad position (%.02f,%.02f) %.02fx%.02f",
1364              sys->picQuad.cropViewport[0].TopLeftX, sys->picQuad.cropViewport[0].TopLeftY,
1365              sys->picQuad.cropViewport[0].Width, sys->picQuad.cropViewport[0].Height );
1366 #endif
1367 }
1368
1369 static bool CanUseTextureArray(vout_display_t *vd)
1370 {
1371 #ifndef HAVE_ID3D11VIDEODECODER
1372     (void) vd;
1373     return false;
1374 #else
1375     struct wddm_version WDDM = {
1376         .wddm         = 22,
1377         .d3d_features = 19,
1378         .revision     = 162,
1379         .build        = 0,
1380     };
1381     if (D3D11CheckDriverVersion(&vd->sys->d3d_dev, GPU_MANUFACTURER_AMD, &WDDM) == VLC_SUCCESS)
1382         return true;
1383
1384     msg_Dbg(vd, "fallback to legacy shader mode for old AMD drivers");
1385     return false;
1386 #endif
1387 }
1388
1389 /* TODO : handle errors better
1390    TODO : seperate out into smaller functions like createshaders */
1391 static int Direct3D11CreateFormatResources(vout_display_t *vd, const video_format_t *fmt)
1392 {
1393     vout_display_sys_t *sys = vd->sys;
1394     HRESULT hr;
1395
1396     sys->legacy_shader = sys->d3d_dev.feature_level < D3D_FEATURE_LEVEL_10_0 || !CanUseTextureArray(vd);
1397
1398     hr = D3D11_CompilePixelShader(vd, &sys->hd3d, sys->legacy_shader, &sys->d3d_dev,
1399                                   sys->picQuad.formatInfo, &sys->display, fmt->transfer, fmt->b_color_range_full,
1400                                   sys->picQuad.d3dpixelShader, sys->picQuad.d3dsampState);
1401     if (FAILED(hr))
1402     {
1403         msg_Err(vd, "Failed to create the pixel shader. (hr=0x%lX)", hr);
1404         return VLC_EGENERIC;
1405     }
1406
1407     sys->picQuad.i_width  = fmt->i_width;
1408     sys->picQuad.i_height = fmt->i_height;
1409     if (!sys->legacy_shader && is_d3d11_opaque(fmt->i_chroma))
1410     {
1411         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x7F) & ~0x7F;
1412         sys->picQuad.i_height = (sys->picQuad.i_height + 0x7F) & ~0x7F;
1413     }
1414     else
1415     if ( sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
1416          sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
1417     {
1418         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x01) & ~0x01;
1419         sys->picQuad.i_height = (sys->picQuad.i_height + 0x01) & ~0x01;
1420     }
1421
1422     UpdateRects(vd, NULL, true);
1423
1424 #ifdef HAVE_ID3D11VIDEODECODER
1425     if (!is_d3d11_opaque(fmt->i_chroma) || sys->legacy_shader)
1426     {
1427         /* we need a staging texture */
1428         ID3D11Texture2D *textures[D3D11_MAX_SHADER_VIEW] = {0};
1429         video_format_t surface_fmt = *fmt;
1430         surface_fmt.i_width  = sys->picQuad.i_width;
1431         surface_fmt.i_height = sys->picQuad.i_height;
1432
1433         if (AllocateTextures(vd, &sys->d3d_dev, sys->picQuad.formatInfo, &surface_fmt, 1, textures))
1434         {
1435             msg_Err(vd, "Failed to allocate the staging texture");
1436             return VLC_EGENERIC;
1437         }
1438
1439         if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->picQuad.formatInfo,
1440                                      textures, 0, sys->stagingSys.resourceView))
1441         {
1442             msg_Err(vd, "Failed to allocate the staging shader view");
1443             return VLC_EGENERIC;
1444         }
1445
1446         for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
1447             sys->stagingSys.texture[plane] = textures[plane];
1448     }
1449 #endif
1450
1451     vd->info.is_slow = !is_d3d11_opaque(fmt->i_chroma) && sys->picQuad.formatInfo->formatTexture != DXGI_FORMAT_UNKNOWN;
1452     return VLC_SUCCESS;
1453 }
1454
1455 static int Direct3D11CreateGenericResources(vout_display_t *vd)
1456 {
1457     vout_display_sys_t *sys = vd->sys;
1458     HRESULT hr;
1459
1460 #if defined(HAVE_ID3D11VIDEODECODER)
1461     sys->d3d_dev.context_mutex = CreateMutexEx( NULL, NULL, 0, SYNCHRONIZE );
1462     ID3D11Device_SetPrivateData( sys->d3d_dev.d3ddevice, &GUID_CONTEXT_MUTEX,
1463                                  sizeof( sys->d3d_dev.context_mutex ), &sys->d3d_dev.context_mutex );
1464 #endif
1465
1466     ID3D11BlendState *pSpuBlendState;
1467     D3D11_BLEND_DESC spuBlendDesc = { 0 };
1468     spuBlendDesc.RenderTarget[0].BlendEnable = TRUE;
1469     spuBlendDesc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
1470     /* output colors */
1471     spuBlendDesc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD;
1472     spuBlendDesc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; /* keep source intact */
1473     spuBlendDesc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; /* RGB colors + inverse alpha (255 is full opaque) */
1474     /* output alpha  */
1475     spuBlendDesc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;
1476     spuBlendDesc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; /* keep source intact */
1477     spuBlendDesc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO; /* discard */
1478
1479     hr = ID3D11Device_CreateBlendState(sys->d3d_dev.d3ddevice, &spuBlendDesc, &pSpuBlendState);
1480     if (FAILED(hr)) {
1481        msg_Err(vd, "Could not create SPU blend state. (hr=0x%lX)", hr);
1482        return VLC_EGENERIC;
1483     }
1484     ID3D11DeviceContext_OMSetBlendState(sys->d3d_dev.d3dcontext, pSpuBlendState, NULL, 0xFFFFFFFF);
1485     ID3D11BlendState_Release(pSpuBlendState);
1486
1487     /* disable depth testing as we're only doing 2D
1488      * see https://msdn.microsoft.com/en-us/library/windows/desktop/bb205074%28v=vs.85%29.aspx
1489      * see http://rastertek.com/dx11tut11.html
1490     */
1491     D3D11_DEPTH_STENCIL_DESC stencilDesc;
1492     ZeroMemory(&stencilDesc, sizeof(stencilDesc));
1493
1494     ID3D11DepthStencilState *pDepthStencilState;
1495     hr = ID3D11Device_CreateDepthStencilState(sys->d3d_dev.d3ddevice, &stencilDesc, &pDepthStencilState );
1496     if (SUCCEEDED(hr)) {
1497         ID3D11DeviceContext_OMSetDepthStencilState(sys->d3d_dev.d3dcontext, pDepthStencilState, 0);
1498         ID3D11DepthStencilState_Release(pDepthStencilState);
1499     }
1500
1501     UpdateRects(vd, NULL, true);
1502
1503     hr = UpdateBackBuffer(vd);
1504     if (FAILED(hr)) {
1505        msg_Err(vd, "Could not update the backbuffer. (hr=0x%lX)", hr);
1506        return VLC_EGENERIC;
1507     }
1508
1509     if (sys->d3dregion_format != NULL)
1510     {
1511         hr = D3D11_CompilePixelShader(vd, &sys->hd3d, sys->legacy_shader, &sys->d3d_dev,
1512                                       sys->d3dregion_format, &sys->display, TRANSFER_FUNC_SRGB, true,
1513                                       sys->pSPUPixelShader, NULL);
1514         if (FAILED(hr))
1515         {
1516             for (size_t i=0; i<D3D11_MAX_SHADER_VIEW; i++)
1517             {
1518                 if (sys->picQuad.d3dpixelShader[i])
1519                 {
1520                     ID3D11PixelShader_Release(sys->picQuad.d3dpixelShader[i]);
1521                     sys->picQuad.d3dpixelShader[i] = NULL;
1522                 }
1523             }
1524             msg_Err(vd, "Failed to create the SPU pixel shader. (hr=0x%lX)", hr);
1525             return VLC_EGENERIC;
1526         }
1527     }
1528
1529     ID3DBlob *pVSBlob = D3D11_CompileShader(vd, &sys->hd3d, &sys->d3d_dev, globVertexShaderFlat, false);
1530     if (!pVSBlob)
1531         return VLC_EGENERIC;
1532
1533     hr = ID3D11Device_CreateVertexShader(sys->d3d_dev.d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1534                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->flatVSShader);
1535
1536     if(FAILED(hr)) {
1537       ID3D10Blob_Release(pVSBlob);
1538       msg_Err(vd, "Failed to create the flat vertex shader. (hr=0x%lX)", hr);
1539       return VLC_EGENERIC;
1540     }
1541
1542     D3D11_INPUT_ELEMENT_DESC layout[] = {
1543     { "POSITION", 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
1544     { "TEXCOORD", 0, DXGI_FORMAT_R32G32_FLOAT,    0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
1545     };
1546
1547     hr = ID3D11Device_CreateInputLayout(sys->d3d_dev.d3ddevice, layout, 2, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1548                                         ID3D10Blob_GetBufferSize(pVSBlob), &sys->pVertexLayout);
1549
1550     ID3D10Blob_Release(pVSBlob);
1551
1552     if(FAILED(hr)) {
1553       msg_Err(vd, "Failed to create the vertex input layout. (hr=0x%lX)", hr);
1554       return VLC_EGENERIC;
1555     }
1556
1557     pVSBlob = D3D11_CompileShader(vd, &sys->hd3d, &sys->d3d_dev, globVertexShaderProjection, false);
1558     if (!pVSBlob)
1559         return VLC_EGENERIC;
1560
1561     hr = ID3D11Device_CreateVertexShader(sys->d3d_dev.d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
1562                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->projectionVSShader);
1563
1564     if(FAILED(hr)) {
1565       ID3D10Blob_Release(pVSBlob);
1566       msg_Err(vd, "Failed to create the projection vertex shader. (hr=0x%lX)", hr);
1567       return VLC_EGENERIC;
1568     }
1569     ID3D10Blob_Release(pVSBlob);
1570
1571     UpdatePicQuadPosition(vd);
1572
1573     msg_Dbg(vd, "Direct3D11 resources created");
1574     return VLC_SUCCESS;
1575 }
1576
1577 static void Direct3D11DestroyPool(vout_display_t *vd)
1578 {
1579     vout_display_sys_t *sys = vd->sys;
1580
1581     if (sys->sys.pool)
1582         picture_pool_Release(sys->sys.pool);
1583     sys->sys.pool = NULL;
1584 }
1585
1586 static void Direct3D11DestroyResources(vout_display_t *vd)
1587 {
1588     vout_display_sys_t *sys = vd->sys;
1589
1590     Direct3D11DestroyPool(vd);
1591
1592     D3D11_ReleaseQuad(&sys->picQuad);
1593     Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
1594     sys->d3dregion_count = 0;
1595
1596     ReleasePictureSys(&sys->stagingSys);
1597
1598     if (sys->pVertexLayout)
1599     {
1600         ID3D11InputLayout_Release(sys->pVertexLayout);
1601         sys->pVertexLayout = NULL;
1602     }
1603     if (sys->flatVSShader)
1604     {
1605         ID3D11VertexShader_Release(sys->flatVSShader);
1606         sys->flatVSShader = NULL;
1607     }
1608     if (sys->projectionVSShader)
1609     {
1610         ID3D11VertexShader_Release(sys->projectionVSShader);
1611         sys->projectionVSShader = NULL;
1612     }
1613     for (size_t i=0; i < D3D11_MAX_SHADER_VIEW; i++)
1614     {
1615         if (sys->d3drenderTargetView[i]) {
1616             ID3D11RenderTargetView_Release(sys->d3drenderTargetView[i]);
1617             sys->d3drenderTargetView[i] = NULL;
1618         }
1619         if (sys->pSPUPixelShader[i])
1620         {
1621             ID3D11PixelShader_Release(sys->pSPUPixelShader[i]);
1622             sys->pSPUPixelShader[i] = NULL;
1623         }
1624     }
1625 #if defined(HAVE_ID3D11VIDEODECODER)
1626     if( sys->d3d_dev.context_mutex != INVALID_HANDLE_VALUE )
1627     {
1628         CloseHandle( sys->d3d_dev.context_mutex );
1629         sys->d3d_dev.context_mutex = INVALID_HANDLE_VALUE;
1630     }
1631 #endif
1632
1633     msg_Dbg(vd, "Direct3D11 resources destroyed");
1634 }
1635
1636 static void Direct3D11DeleteRegions(int count, picture_t **region)
1637 {
1638     for (int i = 0; i < count; ++i) {
1639         if (region[i]) {
1640             picture_Release(region[i]);
1641         }
1642     }
1643     free(region);
1644 }
1645
1646 static void DestroyPictureQuad(picture_t *p_picture)
1647 {
1648     D3D11_ReleaseQuad( (d3d_quad_t *) p_picture->p_sys );
1649     free( p_picture );
1650 }
1651
1652 static int Direct3D11MapSubpicture(vout_display_t *vd, int *subpicture_region_count,
1653                                    picture_t ***region, subpicture_t *subpicture)
1654 {
1655     vout_display_sys_t *sys = vd->sys;
1656     D3D11_MAPPED_SUBRESOURCE mappedResource;
1657     D3D11_TEXTURE2D_DESC texDesc;
1658     HRESULT hr;
1659     int err;
1660
1661     if (sys->d3dregion_format == NULL)
1662         return VLC_EGENERIC;
1663
1664     int count = 0;
1665     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
1666         count++;
1667
1668     *region = calloc(count, sizeof(picture_t *));
1669     if (unlikely(*region==NULL))
1670         return VLC_ENOMEM;
1671     *subpicture_region_count = count;
1672
1673     int i = 0;
1674     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) {
1675         if (!r->fmt.i_visible_width || !r->fmt.i_visible_height)
1676             continue; // won't render anything, keep the cache for later
1677
1678         for (int j = 0; j < sys->d3dregion_count; j++) {
1679             picture_t *cache = sys->d3dregions[j];
1680             if (cache != NULL && ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX]) {
1681                 ID3D11Texture2D_GetDesc( ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX], &texDesc );
1682                 if (texDesc.Format == sys->d3dregion_format->formatTexture &&
1683                     texDesc.Width  == r->p_picture->format.i_width &&
1684                     texDesc.Height == r->p_picture->format.i_height) {
1685                     (*region)[i] = cache;
1686                     memset(&sys->d3dregions[j], 0, sizeof(cache)); // do not reuse this cached value
1687                     break;
1688                 }
1689             }
1690         }
1691
1692         RECT output;
1693         output.left   = r->fmt.i_x_offset;
1694         output.right  = r->fmt.i_x_offset + r->fmt.i_visible_width;
1695         output.top    = r->fmt.i_y_offset;
1696         output.bottom = r->fmt.i_y_offset + r->fmt.i_visible_height;
1697
1698         picture_t *quad_picture = (*region)[i];
1699         if (quad_picture == NULL) {
1700             d3d_quad_t *d3dquad = calloc(1, sizeof(*d3dquad));
1701             if (unlikely(d3dquad==NULL)) {
1702                 continue;
1703             }
1704             if (AllocateTextures(vd, &sys->d3d_dev, sys->d3dregion_format, &r->p_picture->format, 1, d3dquad->picSys.texture)) {
1705                 msg_Err(vd, "Failed to allocate %dx%d texture for OSD",
1706                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1707                 for (int j=0; j<D3D11_MAX_SHADER_VIEW; j++)
1708                     if (d3dquad->picSys.texture[j])
1709                         ID3D11Texture2D_Release(d3dquad->picSys.texture[j]);
1710                 free(d3dquad);
1711                 continue;
1712             }
1713
1714             if (D3D11_AllocateShaderView(vd, sys->d3d_dev.d3ddevice, sys->d3dregion_format,
1715                                          d3dquad->picSys.texture, 0,
1716                                          d3dquad->picSys.resourceView)) {
1717                 msg_Err(vd, "Failed to create %dx%d shader view for OSD",
1718                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1719                 free(d3dquad);
1720                 continue;
1721             }
1722             d3dquad->i_width    = r->fmt.i_width;
1723             d3dquad->i_height   = r->fmt.i_height;
1724
1725             d3dquad->formatInfo = sys->d3dregion_format;
1726             err = D3D11_SetupQuad( vd, &sys->d3d_dev, &r->fmt, d3dquad, &sys->display, &output,
1727                              sys->flatVSShader, sys->pVertexLayout, PROJECTION_MODE_RECTANGULAR, ORIENT_NORMAL );
1728             if (err != VLC_SUCCESS) {
1729                 msg_Err(vd, "Failed to create %dx%d quad for OSD",
1730                         r->fmt.i_visible_width, r->fmt.i_visible_height);
1731                 free(d3dquad);
1732                 continue;
1733             }
1734             picture_resource_t picres = {
1735                 .p_sys      = (picture_sys_t *) d3dquad,
1736                 .pf_destroy = DestroyPictureQuad,
1737             };
1738             (*region)[i] = picture_NewFromResource(&r->p_picture->format, &picres);
1739             if ((*region)[i] == NULL) {
1740                 msg_Err(vd, "Failed to create %dx%d picture for OSD",
1741                         r->fmt.i_width, r->fmt.i_height);
1742                 D3D11_ReleaseQuad(d3dquad);
1743                 continue;
1744             }
1745             quad_picture = (*region)[i];
1746             for (size_t j=0; j<D3D11_MAX_SHADER_VIEW; j++)
1747             {
1748                 /* TODO use something more accurate if we have different formats */
1749                 if (sys->pSPUPixelShader[j])
1750                 {
1751                     d3dquad->d3dpixelShader[j] = sys->pSPUPixelShader[j];
1752                     ID3D11PixelShader_AddRef(d3dquad->d3dpixelShader[j]);
1753                 }
1754             }
1755         } else {
1756             D3D11_UpdateQuadPosition(vd, &sys->d3d_dev, (d3d_quad_t *) quad_picture->p_sys, &output, ORIENT_NORMAL);
1757         }
1758
1759         hr = ID3D11DeviceContext_Map(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
1760         if( SUCCEEDED(hr) ) {
1761             err = CommonUpdatePicture(quad_picture, NULL, mappedResource.pData, mappedResource.RowPitch);
1762             if (err != VLC_SUCCESS) {
1763                 msg_Err(vd, "Failed to set the buffer on the SPU picture" );
1764                 ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
1765                 picture_Release(quad_picture);
1766                 if ((*region)[i] == quad_picture)
1767                     (*region)[i] = NULL;
1768                 continue;
1769             }
1770
1771             picture_CopyPixels(quad_picture, r->p_picture);
1772
1773             ID3D11DeviceContext_Unmap(sys->d3d_dev.d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
1774         } else {
1775             msg_Err(vd, "Failed to map the SPU texture (hr=0x%lX)", hr );
1776             picture_Release(quad_picture);
1777             if ((*region)[i] == quad_picture)
1778                 (*region)[i] = NULL;
1779             continue;
1780         }
1781
1782         d3d_quad_t *quad = (d3d_quad_t *) quad_picture->p_sys;
1783
1784         RECT spuViewport;
1785         spuViewport.left   = sys->sys.rect_dest.left + (FLOAT) r->i_x * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
1786         spuViewport.top    = sys->sys.rect_dest.top  + (FLOAT) r->i_y * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
1787         spuViewport.right  = sys->sys.rect_dest.left + (FLOAT) (r->i_x + r->fmt.i_visible_width)  * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
1788         spuViewport.bottom = sys->sys.rect_dest.top  + (FLOAT) (r->i_y + r->fmt.i_visible_height) * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
1789
1790         D3D11_UpdateViewport( quad, &spuViewport, sys->display.pixelFormat );
1791
1792         D3D11_UpdateQuadOpacity(vd, &sys->d3d_dev, quad, r->i_alpha / 255.0f );
1793     }
1794     return VLC_SUCCESS;
1795 }
1796