auhal: Add debug message to check what audio device is restored
[vlc.git] / modules / video_output / win32 / direct3d11.c
1 /*****************************************************************************
2  * direct3d11.c: Windows Direct3D11 video output module
3  *****************************************************************************
4  * Copyright (C) 2014-2015 VLC authors and VideoLAN
5  *
6  * Authors: Martell Malone <martellmalone@gmail.com>
7  *          Steve Lhomme <robux4@gmail.com>
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU Lesser General Public License as published by
11  * the Free Software Foundation; either version 2.1 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public License
20  * along with this program; if not, write to the Free Software Foundation,
21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < _WIN32_WINNT_WIN7
25 # undef _WIN32_WINNT
26 # define _WIN32_WINNT _WIN32_WINNT_WIN7
27 #endif
28
29 #ifdef HAVE_CONFIG_H
30 # include "config.h"
31 #endif
32
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_vout_display.h>
36
37 #include <assert.h>
38 #include <math.h>
39
40 #define COBJMACROS
41 #include <initguid.h>
42 #include <d3d11.h>
43 #ifdef HAVE_DXGI1_6_H
44 # include <dxgi1_6.h>
45 #else
46 # include <dxgi1_5.h>
47 #endif
48 #include <d3dcompiler.h>
49
50 /* avoided until we can pass ISwapchainPanel without c++/cx mode
51 # include <windows.ui.xaml.media.dxinterop.h> */
52
53 #include "../../video_chroma/d3d11_fmt.h"
54
55 #include "common.h"
56
57 #if !VLC_WINSTORE_APP
58 # define D3D11CreateDevice(args...)             sys->OurD3D11CreateDevice(args)
59 # define D3DCompile(args...)                    sys->OurD3DCompile(args)
60 #endif
61
62 DEFINE_GUID(GUID_SWAPCHAIN_WIDTH,  0xf1b59347, 0x1643, 0x411a, 0xad, 0x6b, 0xc7, 0x80, 0x17, 0x7a, 0x06, 0xb6);
63 DEFINE_GUID(GUID_SWAPCHAIN_HEIGHT, 0x6ea976a0, 0x9d60, 0x4bb7, 0xa5, 0xa9, 0x7d, 0xd1, 0x18, 0x7f, 0xc9, 0xbd);
64
65 static int  Open(vlc_object_t *);
66 static void Close(vlc_object_t *);
67
68 #define DEFAULT_BRIGHTNESS         100
69 #define DEFAULT_SRGB_BRIGHTNESS    100
70 #define MAX_PQ_BRIGHTNESS        10000
71
72 #define D3D11_HELP N_("Recommended video output for Windows 8 and later versions")
73 #define HW_BLENDING_TEXT N_("Use hardware blending support")
74 #define HW_BLENDING_LONGTEXT N_(\
75     "Try to use hardware acceleration for subtitle/OSD blending.")
76
77 vlc_module_begin ()
78     set_shortname("Direct3D11")
79     set_description(N_("Direct3D11 video output"))
80     set_help(D3D11_HELP)
81     set_category(CAT_VIDEO)
82     set_subcategory(SUBCAT_VIDEO_VOUT)
83
84     add_bool("direct3d11-hw-blending", true, HW_BLENDING_TEXT, HW_BLENDING_LONGTEXT, true)
85
86 #if VLC_WINSTORE_APP
87     add_integer("winrt-d3dcontext",    0x0, NULL, NULL, true); /* ID3D11DeviceContext* */
88     add_integer("winrt-swapchain",     0x0, NULL, NULL, true); /* IDXGISwapChain1*     */
89 #endif
90
91     set_capability("vout display", 300)
92     add_shortcut("direct3d11")
93     set_callbacks(Open, Close)
94 vlc_module_end ()
95
96 /* A Quad is texture that can be displayed in a rectangle */
97 typedef struct
98 {
99     picture_sys_t             picSys;
100     ID3D11Buffer              *pVertexBuffer;
101     UINT                      vertexCount;
102     ID3D11VertexShader        *d3dvertexShader;
103     ID3D11Buffer              *pIndexBuffer;
104     UINT                      indexCount;
105     ID3D11Buffer              *pVertexShaderConstants;
106     ID3D11Buffer              *pPixelShaderConstants[2];
107     UINT                       PSConstantsCount;
108     ID3D11PixelShader         *d3dpixelShader;
109     D3D11_VIEWPORT            cropViewport;
110     const vlc_chroma_description_t *p_chroma_sampling;
111     unsigned int              i_width;
112     unsigned int              i_height;
113 } d3d_quad_t;
114
115 typedef enum video_color_axis {
116     COLOR_AXIS_RGB,
117     COLOR_AXIS_YCBCR,
118 } video_color_axis;
119
120 typedef struct {
121     DXGI_COLOR_SPACE_TYPE   dxgi;
122     const char              *name;
123     video_color_axis        axis;
124     video_color_primaries_t primaries;
125     video_transfer_func_t   transfer;
126     video_color_space_t     color;
127     bool                    b_full_range;
128 } dxgi_color_space;
129
130 struct vout_display_sys_t
131 {
132     vout_display_sys_win32_t sys;
133
134     struct { /* TODO may go in vout_display_info_t without DXGI */
135         const dxgi_color_space   *colorspace;
136         unsigned                 luminance_peak;
137     } display;
138
139 #if !VLC_WINSTORE_APP
140     HINSTANCE                hdxgi_dll;        /* handle of the opened dxgi dll */
141     HINSTANCE                hd3d11_dll;       /* handle of the opened d3d11 dll */
142     HINSTANCE                hd3dcompiler_dll; /* handle of the opened d3dcompiler dll */
143     /* We should find a better way to store this or atleast a shorter name */
144     PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN OurD3D11CreateDeviceAndSwapChain;
145     PFN_D3D11_CREATE_DEVICE                OurD3D11CreateDevice;
146     pD3DCompile                            OurD3DCompile;
147 #endif
148 #if defined(HAVE_ID3D11VIDEODECODER)
149     HANDLE                   context_lock;     /* D3D11 Context lock necessary
150                                                   for hw decoding */
151 #endif
152     IDXGISwapChain1          *dxgiswapChain;   /* DXGI 1.2 swap chain */
153     IDXGISwapChain4          *dxgiswapChain4;  /* DXGI 1.5 for HDR */
154     ID3D11Device             *d3ddevice;       /* D3D device */
155     ID3D11DeviceContext      *d3dcontext;      /* D3D context */
156     d3d_quad_t               picQuad;
157     const d3d_format_t       *picQuadConfig;
158     ID3D11PixelShader        *picQuadPixelShader;
159
160     picture_sys_t            stagingSys;
161
162     ID3D11RenderTargetView   *d3drenderTargetView;
163     ID3D11DepthStencilView   *d3ddepthStencilView;
164
165     ID3D11VertexShader        *flatVSShader;
166     ID3D11VertexShader        *projectionVSShader;
167
168     /* copy from the decoder pool into picSquad before display
169      * Uses a Texture2D with slices rather than a Texture2DArray for the decoder */
170     bool                     legacy_shader;
171
172     // SPU
173     vlc_fourcc_t             pSubpictureChromas[2];
174     ID3D11PixelShader        *pSPUPixelShader;
175     const d3d_format_t       *d3dregion_format;
176     int                      d3dregion_count;
177     picture_t                **d3dregions;
178 };
179
180 /* matches the D3D11_INPUT_ELEMENT_DESC we setup */
181 typedef struct d3d_vertex_t {
182     struct {
183         FLOAT x;
184         FLOAT y;
185         FLOAT z;
186     } position;
187     struct {
188         FLOAT x;
189         FLOAT y;
190     } texture;
191 } d3d_vertex_t;
192
193 typedef struct {
194     FLOAT Opacity;
195     FLOAT opacityPadding[3];
196 } PS_CONSTANT_BUFFER;
197
198 typedef struct {
199     FLOAT WhitePoint[4*4];
200     FLOAT Colorspace[4*4];
201 } PS_COLOR_TRANSFORM;
202
203 typedef struct {
204     FLOAT RotX[4*4];
205     FLOAT RotY[4*4];
206     FLOAT RotZ[4*4];
207     FLOAT View[4*4];
208     FLOAT Projection[4*4];
209 } VS_PROJECTION_CONST;
210
211 #define SPHERE_RADIUS 1.f
212
213 #define RECTWidth(r)   (int)((r).right - (r).left)
214 #define RECTHeight(r)  (int)((r).bottom - (r).top)
215
216 static picture_pool_t *Pool(vout_display_t *vd, unsigned count);
217
218 static void Prepare(vout_display_t *, picture_t *, subpicture_t *subpicture);
219 static void Display(vout_display_t *, picture_t *, subpicture_t *subpicture);
220
221 static HINSTANCE Direct3D11LoadShaderLibrary(void);
222 static void Direct3D11Destroy(vout_display_t *);
223
224 static int  Direct3D11Open (vout_display_t *, video_format_t *);
225 static void Direct3D11Close(vout_display_t *);
226
227 static int  Direct3D11CreateResources (vout_display_t *, video_format_t *);
228 static void Direct3D11DestroyResources(vout_display_t *);
229
230 static void Direct3D11DestroyPool(vout_display_t *);
231
232 static void DestroyDisplayPoolPicture(picture_t *);
233 static void Direct3D11DeleteRegions(int, picture_t **);
234 static int Direct3D11MapSubpicture(vout_display_t *, int *, picture_t ***, subpicture_t *);
235
236 static int SetupQuad(vout_display_t *, const video_format_t *, d3d_quad_t *, const RECT *,
237                      const d3d_format_t *, ID3D11PixelShader *, video_projection_mode_t,
238                      video_orientation_t);
239 static bool UpdateQuadPosition( vout_display_t *vd, d3d_quad_t *quad,
240                                 const RECT *output,
241                                 video_projection_mode_t projection,
242                                 video_orientation_t orientation );
243 static void ReleaseQuad(d3d_quad_t *);
244 static void UpdatePicQuadPosition(vout_display_t *);
245
246 static int Control(vout_display_t *vd, int query, va_list args);
247 static void Manage(vout_display_t *vd);
248
249 /* TODO: Move to a direct3d11_shaders header */
250 static const char* globVertexShaderFlat = "\
251   struct VS_INPUT\
252   {\
253     float4 Position   : POSITION;\
254     float4 Texture    : TEXCOORD0;\
255   };\
256   \
257   struct VS_OUTPUT\
258   {\
259     float4 Position   : SV_POSITION;\
260     float4 Texture    : TEXCOORD0;\
261   };\
262   \
263   VS_OUTPUT main( VS_INPUT In )\
264   {\
265     return In;\
266   }\
267 ";
268
269 #define STRINGIZE2(s) #s
270 #define STRINGIZE(s) STRINGIZE2(s)
271
272 static const char* globVertexShaderProjection = "\
273   cbuffer VS_PROJECTION_CONST : register(b0)\
274   {\
275      float4x4 RotX;\
276      float4x4 RotY;\
277      float4x4 RotZ;\
278      float4x4 View;\
279      float4x4 Projection;\
280   };\
281   struct VS_INPUT\
282   {\
283     float4 Position   : POSITION;\
284     float4 Texture    : TEXCOORD0;\
285   };\
286   \
287   struct VS_OUTPUT\
288   {\
289     float4 Position   : SV_POSITION;\
290     float4 Texture    : TEXCOORD0;\
291   };\
292   \
293   VS_OUTPUT main( VS_INPUT In )\
294   {\
295     VS_OUTPUT Output;\
296     float4 pos = In.Position;\
297     pos = mul(RotY, pos);\
298     pos = mul(RotX, pos);\
299     pos = mul(RotZ, pos);\
300     pos = mul(View, pos);\
301     pos = mul(Projection, pos);\
302     Output.Position = pos;\
303     Output.Texture = In.Texture;\
304     return Output;\
305   }\
306 ";
307
308 static const char* globPixelShaderDefault = "\
309   cbuffer PS_CONSTANT_BUFFER : register(b0)\
310   {\
311     float Opacity;\
312     float opacityPadding[3];\
313   };\
314   cbuffer PS_COLOR_TRANSFORM : register(b1)\
315   {\
316     float4x4 WhitePoint;\
317     float4x4 Colorspace;\
318   };\
319   Texture2D%s shaderTexture[" STRINGIZE(D3D11_MAX_SHADER_VIEW) "];\
320   SamplerState SampleType;\
321   \
322   struct PS_INPUT\
323   {\
324     float4 Position   : SV_POSITION;\
325     float4 Texture    : TEXCOORD0;\
326   };\
327   \
328   /* see http://filmicworlds.com/blog/filmic-tonemapping-operators/ */\
329   inline float hable(float x) {\
330       const float A = 0.15, B = 0.50, C = 0.10, D = 0.20, E = 0.02, F = 0.30;\
331       return ((x * (A*x + (C*B))+(D*E))/(x * (A*x + B) + (D*F))) - E/F;\
332   }\
333   \
334   inline float3 hable(float3 x) {\
335       x.r = hable(x.r);\
336       x.g = hable(x.g);\
337       x.b = hable(x.b);\
338       return x;\
339   }\
340   \
341   /* https://en.wikipedia.org/wiki/Hybrid_Log-Gamma#Technical_details */\
342   inline float inverse_HLG(float x){\
343       const float B67_a = 0.17883277;\
344       const float B67_b = 0.28466892;\
345       const float B67_c = 0.55991073;\
346       const float B67_inv_r2 = 4.0; /* 1/0.5² */\
347       if (x <= 0.5)\
348           x = x * x * B67_inv_r2;\
349       else\
350           x = exp((x - B67_c) / B67_a) + B67_b;\
351       return x;\
352   }\
353   \
354   inline float3 sourceToLinear(float3 rgb) {\
355       %s;\
356   }\
357   \
358   inline float3 linearToDisplay(float3 rgb) {\
359       %s;\
360   }\
361   \
362   inline float3 toneMapping(float3 rgb) {\
363       %s;\
364   }\
365   \
366   inline float3 adjustPeakLuminance(float3 rgb) {\
367       %s;\
368   }\
369   \
370   inline float3 adjustRange(float3 rgb) {\
371       %s;\
372   }\
373   \
374   float4 main( PS_INPUT In ) : SV_TARGET\
375   {\
376     float4 sample;\
377     \
378     %s /* sampling routine in sample */\
379     float4 rgba = mul(mul(sample, WhitePoint), Colorspace);\
380     float opacity = rgba.a * Opacity;\
381     float3 rgb = (float3)rgba;\
382     rgb = sourceToLinear(rgb);\
383     rgb = toneMapping(rgb);\
384     rgb = adjustPeakLuminance(rgb);\
385     rgb = linearToDisplay(rgb);\
386     rgb = adjustRange(rgb);\
387     return float4(rgb, saturate(opacity));\
388   }\
389 ";
390
391 #define ST2084_PQ_CONSTANTS  "const float ST2084_m1 = 2610.0 / (4096.0 * 4);\
392 const float ST2084_m2 = (2523.0 / 4096.0) * 128.0;\
393 const float ST2084_c1 = 3424.0 / 4096.0;\
394 const float ST2084_c2 = (2413.0 / 4096.0) * 32.0;\
395 const float ST2084_c3 = (2392.0 / 4096.0) * 32.0;"
396
397 static int Direct3D11MapPoolTexture(picture_t *picture)
398 {
399     picture_sys_t *p_sys = picture->p_sys;
400     D3D11_MAPPED_SUBRESOURCE mappedResource;
401     HRESULT hr;
402     hr = ID3D11DeviceContext_Map(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
403     if( FAILED(hr) )
404     {
405         return VLC_EGENERIC;
406     }
407     return CommonUpdatePicture(picture, NULL, mappedResource.pData, mappedResource.RowPitch);
408 }
409
410 static void Direct3D11UnmapPoolTexture(picture_t *picture)
411 {
412     picture_sys_t *p_sys = picture->p_sys;
413     ID3D11DeviceContext_Unmap(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0);
414 }
415
416 #if !VLC_WINSTORE_APP
417 static int OpenHwnd(vout_display_t *vd)
418 {
419     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
420     if (!sys)
421         return VLC_ENOMEM;
422
423     sys->hd3d11_dll = LoadLibrary(TEXT("D3D11.DLL"));
424     if (!sys->hd3d11_dll) {
425         msg_Warn(vd, "cannot load d3d11.dll, aborting");
426         return VLC_EGENERIC;
427     }
428
429     sys->hd3dcompiler_dll = Direct3D11LoadShaderLibrary();
430     if (!sys->hd3dcompiler_dll) {
431         msg_Err(vd, "cannot load d3dcompiler.dll, aborting");
432         Direct3D11Destroy(vd);
433         return VLC_EGENERIC;
434     }
435
436     sys->OurD3DCompile = (void *)GetProcAddress(sys->hd3dcompiler_dll, "D3DCompile");
437     if (!sys->OurD3DCompile) {
438         msg_Err(vd, "Cannot locate reference to D3DCompile in d3dcompiler DLL");
439         Direct3D11Destroy(vd);
440         return VLC_EGENERIC;
441     }
442
443     sys->OurD3D11CreateDevice =
444         (void *)GetProcAddress(sys->hd3d11_dll, "D3D11CreateDevice");
445     if (!sys->OurD3D11CreateDevice) {
446         msg_Err(vd, "Cannot locate reference to D3D11CreateDevice in d3d11 DLL");
447         Direct3D11Destroy(vd);
448         return VLC_EGENERIC;
449     }
450     return VLC_SUCCESS;
451 }
452 #else
453 static int OpenCoreW(vout_display_t *vd)
454 {
455     IDXGISwapChain1* dxgiswapChain  = var_InheritInteger(vd, "winrt-swapchain");
456     if (!dxgiswapChain)
457         return VLC_EGENERIC;
458     ID3D11DeviceContext* d3dcontext = var_InheritInteger(vd, "winrt-d3dcontext");
459     if (!d3dcontext)
460         return VLC_EGENERIC;
461     ID3D11Device* d3ddevice = NULL;
462     ID3D11DeviceContext_GetDevice(d3dcontext, &d3ddevice);
463     if (!d3ddevice)
464         return VLC_EGENERIC;
465
466     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
467     if (!sys)
468         return VLC_ENOMEM;
469
470     sys->dxgiswapChain = dxgiswapChain;
471     sys->d3ddevice     = d3ddevice;
472     sys->d3dcontext    = d3dcontext;
473     IDXGISwapChain_AddRef     (sys->dxgiswapChain);
474     ID3D11Device_AddRef       (sys->d3ddevice);
475     ID3D11DeviceContext_AddRef(sys->d3dcontext);
476
477     return VLC_SUCCESS;
478 }
479 #endif
480
481 static bool is_d3d11_opaque(vlc_fourcc_t chroma)
482 {
483     switch (chroma)
484     {
485     case VLC_CODEC_D3D11_OPAQUE:
486     case VLC_CODEC_D3D11_OPAQUE_10B:
487         return true;
488     default:
489         return false;
490     }
491 }
492
493 #if VLC_WINSTORE_APP
494 static bool GetRect(const vout_display_sys_win32_t *p_sys, RECT *out)
495 {
496     const vout_display_sys_t *sys = (const vout_display_sys_t *)p_sys;
497     out->left   = 0;
498     out->top    = 0;
499     uint32_t i_width;
500     uint32_t i_height;
501     UINT dataSize = sizeof(i_width);
502     HRESULT hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_WIDTH, &dataSize, &i_width);
503     if (FAILED(hr)) {
504         return false;
505     }
506     dataSize = sizeof(i_height);
507     hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_HEIGHT, &dataSize, &i_height);
508     if (FAILED(hr)) {
509         return false;
510     }
511     out->right  = i_width;
512     out->bottom = i_height;
513     return true;
514 }
515 #endif
516
517 static int Open(vlc_object_t *object)
518 {
519     vout_display_t *vd = (vout_display_t *)object;
520
521 #if !VLC_WINSTORE_APP
522     int ret = OpenHwnd(vd);
523 #else
524     int ret = OpenCoreW(vd);
525 #endif
526
527     if (ret != VLC_SUCCESS)
528         return ret;
529
530     if (CommonInit(vd))
531         goto error;
532
533 #if VLC_WINSTORE_APP
534     vd->sys->sys.pf_GetRect = GetRect;
535 #endif
536
537     video_format_t fmt;
538     video_format_Copy(&fmt, &vd->source);
539     if (Direct3D11Open(vd, &fmt)) {
540         msg_Err(vd, "Direct3D11 could not be opened");
541         goto error;
542     }
543
544     video_format_Clean(&vd->fmt);
545     vd->fmt = fmt;
546
547     vd->info.has_double_click     = true;
548     vd->info.has_pictures_invalid = vd->info.is_slow;
549
550     if (var_InheritBool(vd, "direct3d11-hw-blending") &&
551         vd->sys->d3dregion_format != NULL)
552     {
553         vd->sys->pSubpictureChromas[0] = vd->sys->d3dregion_format->fourcc;
554         vd->sys->pSubpictureChromas[1] = 0;
555         vd->info.subpicture_chromas = vd->sys->pSubpictureChromas;
556     }
557     else
558         vd->info.subpicture_chromas = NULL;
559
560     vd->pool    = Pool;
561     vd->prepare = Prepare;
562     vd->display = Display;
563     vd->control = Control;
564     vd->manage  = Manage;
565
566     msg_Dbg(vd, "Direct3D11 Open Succeeded");
567
568     return VLC_SUCCESS;
569
570 error:
571     Close(object);
572     return VLC_EGENERIC;
573 }
574
575 static void Close(vlc_object_t *object)
576 {
577     vout_display_t * vd = (vout_display_t *)object;
578
579     Direct3D11Close(vd);
580     CommonClean(vd);
581     Direct3D11Destroy(vd);
582     free(vd->sys);
583 }
584
585 static int AllocateTextures(vout_display_t *vd, const d3d_format_t *cfg,
586                             const video_format_t *fmt, unsigned pool_size,
587                             ID3D11Texture2D *textures[])
588 {
589     vout_display_sys_t *sys = vd->sys;
590     plane_t planes[PICTURE_PLANE_MAX];
591     int plane, plane_count;
592     HRESULT hr;
593     ID3D11Texture2D *slicedTexture = NULL;
594     D3D11_TEXTURE2D_DESC texDesc;
595     ZeroMemory(&texDesc, sizeof(texDesc));
596     texDesc.MipLevels = 1;
597     texDesc.SampleDesc.Count = 1;
598     texDesc.MiscFlags = 0; //D3D11_RESOURCE_MISC_SHARED;
599     texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
600     if (is_d3d11_opaque(fmt->i_chroma)) {
601         texDesc.BindFlags |= D3D11_BIND_DECODER;
602         texDesc.Usage = D3D11_USAGE_DEFAULT;
603         texDesc.CPUAccessFlags = 0;
604     } else {
605         texDesc.Usage = D3D11_USAGE_DYNAMIC;
606         texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
607     }
608     texDesc.ArraySize = pool_size;
609
610     const vlc_chroma_description_t *p_chroma_desc = vlc_fourcc_GetChromaDescription( fmt->i_chroma );
611     if( !p_chroma_desc )
612         return VLC_EGENERIC;
613
614     if (cfg->formatTexture == DXGI_FORMAT_UNKNOWN) {
615         if (p_chroma_desc->plane_count == 0)
616         {
617             msg_Dbg(vd, "failed to get the pixel format planes for %4.4s", (char *)&fmt->i_chroma);
618             return VLC_EGENERIC;
619         }
620         assert(p_chroma_desc->plane_count <= D3D11_MAX_SHADER_VIEW);
621         plane_count = p_chroma_desc->plane_count;
622
623         texDesc.Format = cfg->resourceFormat[0];
624         assert(cfg->resourceFormat[1] == cfg->resourceFormat[0]);
625         assert(cfg->resourceFormat[2] == cfg->resourceFormat[0]);
626
627         for( int i = 0; i < plane_count; i++ )
628         {
629             plane_t *p = &planes[i];
630
631             p->i_lines         = fmt->i_height * p_chroma_desc->p[i].h.num / p_chroma_desc->p[i].h.den;
632             p->i_visible_lines = fmt->i_visible_height * p_chroma_desc->p[i].h.num / p_chroma_desc->p[i].h.den;
633             p->i_pitch         = fmt->i_width * p_chroma_desc->p[i].w.num / p_chroma_desc->p[i].w.den * p_chroma_desc->pixel_size;
634             p->i_visible_pitch = fmt->i_visible_width * p_chroma_desc->p[i].w.num / p_chroma_desc->p[i].w.den * p_chroma_desc->pixel_size;
635             p->i_pixel_pitch   = p_chroma_desc->pixel_size;
636         }
637     } else {
638         plane_count = 1;
639         texDesc.Format = cfg->formatTexture;
640         texDesc.Height = fmt->i_height;
641         texDesc.Width = fmt->i_width;
642
643         hr = ID3D11Device_CreateTexture2D( sys->d3ddevice, &texDesc, NULL, &slicedTexture );
644         if (FAILED(hr)) {
645             msg_Err(vd, "CreateTexture2D failed for the %d pool. (hr=0x%0lx)", pool_size, hr);
646             goto error;
647         }
648     }
649
650     for (unsigned picture_count = 0; picture_count < pool_size; picture_count++) {
651         for (plane = 0; plane < plane_count; plane++)
652         {
653             if (slicedTexture) {
654                 textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] = slicedTexture;
655                 ID3D11Texture2D_AddRef(slicedTexture);
656             } else {
657                 texDesc.Height = planes[plane].i_lines;
658                 texDesc.Width = planes[plane].i_pitch;
659                 hr = ID3D11Device_CreateTexture2D( sys->d3ddevice, &texDesc, NULL, &textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] );
660                 if (FAILED(hr)) {
661                     msg_Err(vd, "CreateTexture2D failed for the %d pool. (hr=0x%0lx)", pool_size, hr);
662                     goto error;
663                 }
664             }
665         }
666         for (; plane < D3D11_MAX_SHADER_VIEW; plane++) {
667             if (!cfg->resourceFormat[plane])
668                 textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] = NULL;
669             else
670             {
671                 textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW];
672                 ID3D11Texture2D_AddRef(textures[picture_count * D3D11_MAX_SHADER_VIEW + plane]);
673             }
674         }
675     }
676
677     if (!is_d3d11_opaque(fmt->i_chroma) && cfg->formatTexture != DXGI_FORMAT_UNKNOWN) {
678         D3D11_MAPPED_SUBRESOURCE mappedResource;
679         hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource*)textures[0], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
680         if( FAILED(hr) ) {
681             msg_Err(vd, "The texture cannot be mapped. (hr=0x%lX)", hr);
682             goto error;
683         }
684         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource*)textures[0], 0);
685         if (mappedResource.RowPitch < p_chroma_desc->pixel_size * texDesc.Width) {
686             msg_Err( vd, "The texture row pitch is too small (%d instead of %d)", mappedResource.RowPitch,
687                      p_chroma_desc->pixel_size * texDesc.Width );
688             goto error;
689         }
690     }
691
692     if (slicedTexture)
693         ID3D11Texture2D_Release(slicedTexture);
694     return VLC_SUCCESS;
695 error:
696     if (slicedTexture)
697         ID3D11Texture2D_Release(slicedTexture);
698     return VLC_EGENERIC;
699 }
700
701 static picture_pool_t *Pool(vout_display_t *vd, unsigned pool_size)
702 {
703     /* compensate for extra hardware decoding pulling extra pictures from our pool */
704     pool_size += 2;
705
706     vout_display_sys_t *sys = vd->sys;
707     ID3D11Texture2D  *textures[pool_size * D3D11_MAX_SHADER_VIEW];
708     picture_t **pictures = NULL;
709     picture_t *picture;
710     unsigned  picture_count = 0;
711
712     if (sys->sys.pool)
713         return sys->sys.pool;
714
715     if (vd->info.is_slow)
716         pool_size = 1;
717
718     video_format_t surface_fmt = vd->fmt;
719     surface_fmt.i_width  = sys->picQuad.i_width;
720     surface_fmt.i_height = sys->picQuad.i_height;
721
722     if (!vd->info.is_slow) {
723         HRESULT           hr;
724         ID3D10Multithread *pMultithread;
725         hr = ID3D11Device_QueryInterface( sys->d3ddevice, &IID_ID3D10Multithread, (void **)&pMultithread);
726         if (SUCCEEDED(hr)) {
727             ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
728             ID3D10Multithread_Release(pMultithread);
729         }
730     }
731
732     if (sys->picQuadConfig->formatTexture != DXGI_FORMAT_UNKNOWN)
733     {
734         if (AllocateTextures(vd, sys->picQuadConfig, &surface_fmt, pool_size, textures))
735             goto error;
736
737         pictures = calloc(pool_size, sizeof(*pictures));
738         if (!pictures)
739             goto error;
740
741         for (picture_count = 0; picture_count < pool_size; picture_count++) {
742             picture_sys_t *picsys = calloc(1, sizeof(*picsys));
743             if (unlikely(picsys == NULL))
744                 goto error;
745
746             for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
747                 picsys->texture[plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW + plane];
748
749             picsys->slice_index = picture_count;
750             picsys->formatTexture = sys->picQuadConfig->formatTexture;
751             picsys->context = sys->d3dcontext;
752
753             picture_resource_t resource = {
754                 .p_sys = picsys,
755                 .pf_destroy = DestroyDisplayPoolPicture,
756             };
757
758             picture = picture_NewFromResource(&surface_fmt, &resource);
759             if (unlikely(picture == NULL)) {
760                 free(picsys);
761                 msg_Err( vd, "Failed to create picture %d in the pool.", picture_count );
762                 goto error;
763             }
764
765             pictures[picture_count] = picture;
766             /* each picture_t holds a ref to the context and release it on Destroy */
767             ID3D11DeviceContext_AddRef(picsys->context);
768         }
769     }
770
771 #ifdef HAVE_ID3D11VIDEODECODER
772     if (!is_d3d11_opaque(surface_fmt.i_chroma) || sys->legacy_shader)
773     {
774         /* we need a staging texture */
775         if (AllocateTextures(vd, sys->picQuadConfig, &surface_fmt, 1, textures))
776             goto error;
777
778         sys->picQuad.p_chroma_sampling = vlc_fourcc_GetChromaDescription( surface_fmt.i_chroma );
779
780         for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
781             sys->stagingSys.texture[plane] = textures[plane];
782
783         if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->picQuadConfig,
784                                textures, 0, sys->stagingSys.resourceView))
785             goto error;
786     } else
787 #endif
788     {
789         for (picture_count = 0; picture_count < pool_size; picture_count++) {
790             if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->picQuadConfig,
791                                    pictures[picture_count]->p_sys->texture, picture_count,
792                                    pictures[picture_count]->p_sys->resourceView))
793                 goto error;
794         }
795     }
796
797     if (SetupQuad( vd, &surface_fmt, &sys->picQuad, &sys->sys.rect_src_clipped,
798                    sys->picQuadConfig, sys->picQuadPixelShader,
799                    surface_fmt.projection_mode, vd->fmt.orientation ) != VLC_SUCCESS) {
800         msg_Err(vd, "Could not Create the main quad picture.");
801         return NULL;
802     }
803
804     if (sys->picQuadConfig->formatTexture == DXGI_FORMAT_UNKNOWN)
805         sys->sys.pool = picture_pool_NewFromFormat( &surface_fmt, pool_size );
806     else
807     {
808         picture_pool_configuration_t pool_cfg = {
809             .picture       = pictures,
810             .picture_count = pool_size,
811         };
812         if (vd->info.is_slow) {
813             pool_cfg.lock          = Direct3D11MapPoolTexture;
814             //pool_cfg.unlock        = Direct3D11UnmapPoolTexture;
815         }
816         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
817     }
818
819 error:
820     if (sys->sys.pool == NULL) {
821         picture_pool_configuration_t pool_cfg = {
822             .picture_count = 0,
823         };
824         if (pictures) {
825             msg_Dbg(vd, "Failed to create the picture d3d11 pool");
826             for (unsigned i=0;i<picture_count; ++i)
827                 picture_Release(pictures[i]);
828             free(pictures);
829         }
830
831         /* create an empty pool to avoid crashing */
832         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
833     } else {
834         msg_Dbg(vd, "D3D11 pool succeed with %d surfaces (%dx%d) context 0x%p",
835                 picture_count, surface_fmt.i_width, surface_fmt.i_height, sys->d3dcontext);
836     }
837     return sys->sys.pool;
838 }
839
840 static void DestroyDisplayPoolPicture(picture_t *picture)
841 {
842     picture_sys_t *p_sys = picture->p_sys;
843     ReleasePictureSys( p_sys );
844     free(p_sys);
845     free(picture);
846 }
847
848 static HRESULT UpdateBackBuffer(vout_display_t *vd)
849 {
850     vout_display_sys_t *sys = vd->sys;
851     HRESULT hr;
852     ID3D11Texture2D* pDepthStencil;
853     ID3D11Texture2D* pBackBuffer;
854     RECT rect;
855 #if VLC_WINSTORE_APP
856     if (!GetRect(&sys->sys, &rect))
857 #endif
858         rect = sys->sys.rect_dest_clipped;
859     uint32_t i_width = RECTWidth(rect);
860     uint32_t i_height = RECTHeight(rect);
861
862     if (sys->d3drenderTargetView) {
863         ID3D11RenderTargetView_Release(sys->d3drenderTargetView);
864         sys->d3drenderTargetView = NULL;
865     }
866     if (sys->d3ddepthStencilView) {
867         ID3D11DepthStencilView_Release(sys->d3ddepthStencilView);
868         sys->d3ddepthStencilView = NULL;
869     }
870
871     /* TODO detect is the size is the same as the output and switch to fullscreen mode */
872     hr = IDXGISwapChain_ResizeBuffers(sys->dxgiswapChain, 0, i_width, i_height,
873         DXGI_FORMAT_UNKNOWN, 0);
874     if (FAILED(hr)) {
875        msg_Err(vd, "Failed to resize the backbuffer. (hr=0x%lX)", hr);
876        return hr;
877     }
878
879     hr = IDXGISwapChain_GetBuffer(sys->dxgiswapChain, 0, &IID_ID3D11Texture2D, (LPVOID *)&pBackBuffer);
880     if (FAILED(hr)) {
881        msg_Err(vd, "Could not get the backbuffer for the Swapchain. (hr=0x%lX)", hr);
882        return hr;
883     }
884
885     hr = ID3D11Device_CreateRenderTargetView(sys->d3ddevice, (ID3D11Resource *)pBackBuffer, NULL, &sys->d3drenderTargetView);
886     ID3D11Texture2D_Release(pBackBuffer);
887     if (FAILED(hr)) {
888         msg_Err(vd, "Failed to create the target view. (hr=0x%lX)", hr);
889         return hr;
890     }
891
892     D3D11_TEXTURE2D_DESC deptTexDesc;
893     memset(&deptTexDesc, 0,sizeof(deptTexDesc));
894     deptTexDesc.ArraySize = 1;
895     deptTexDesc.BindFlags = D3D11_BIND_DEPTH_STENCIL;
896     deptTexDesc.CPUAccessFlags = 0;
897     deptTexDesc.Format = DXGI_FORMAT_D24_UNORM_S8_UINT;
898     deptTexDesc.Width = i_width;
899     deptTexDesc.Height = i_height;
900     deptTexDesc.MipLevels = 1;
901     deptTexDesc.MiscFlags = 0;
902     deptTexDesc.SampleDesc.Count = 1;
903     deptTexDesc.SampleDesc.Quality = 0;
904     deptTexDesc.Usage = D3D11_USAGE_DEFAULT;
905
906     hr = ID3D11Device_CreateTexture2D(sys->d3ddevice, &deptTexDesc, NULL, &pDepthStencil);
907     if (FAILED(hr)) {
908        msg_Err(vd, "Could not create the depth stencil texture. (hr=0x%lX)", hr);
909        return hr;
910     }
911
912     D3D11_DEPTH_STENCIL_VIEW_DESC depthViewDesc;
913     memset(&depthViewDesc, 0, sizeof(depthViewDesc));
914
915     depthViewDesc.Format = deptTexDesc.Format;
916     depthViewDesc.ViewDimension = D3D11_DSV_DIMENSION_TEXTURE2D;
917     depthViewDesc.Texture2D.MipSlice = 0;
918
919     hr = ID3D11Device_CreateDepthStencilView(sys->d3ddevice, (ID3D11Resource *)pDepthStencil, &depthViewDesc, &sys->d3ddepthStencilView);
920     ID3D11Texture2D_Release(pDepthStencil);
921
922     if (FAILED(hr)) {
923        msg_Err(vd, "Could not create the depth stencil view. (hr=0x%lX)", hr);
924        return hr;
925     }
926
927     return S_OK;
928 }
929
930 /* rotation around the Z axis */
931 static void getZRotMatrix(float theta, FLOAT matrix[static 16])
932 {
933     float st, ct;
934
935     sincosf(theta, &st, &ct);
936
937     const FLOAT m[] = {
938     /*  x    y    z    w */
939         ct,  -st, 0.f, 0.f,
940         st,  ct,  0.f, 0.f,
941         0.f, 0.f, 1.f, 0.f,
942         0.f, 0.f, 0.f, 1.f
943     };
944
945     memcpy(matrix, m, sizeof(m));
946 }
947
948 /* rotation around the Y axis */
949 static void getYRotMatrix(float theta, FLOAT matrix[static 16])
950 {
951     float st, ct;
952
953     sincosf(theta, &st, &ct);
954
955     const FLOAT m[] = {
956     /*  x    y    z    w */
957         ct,  0.f, -st, 0.f,
958         0.f, 1.f, 0.f, 0.f,
959         st,  0.f, ct,  0.f,
960         0.f, 0.f, 0.f, 1.f
961     };
962
963     memcpy(matrix, m, sizeof(m));
964 }
965
966 /* rotation around the X axis */
967 static void getXRotMatrix(float phi, FLOAT matrix[static 16])
968 {
969     float sp, cp;
970
971     sincosf(phi, &sp, &cp);
972
973     const FLOAT m[] = {
974     /*  x    y    z    w */
975         1.f, 0.f, 0.f, 0.f,
976         0.f, cp,  sp,  0.f,
977         0.f, -sp, cp,  0.f,
978         0.f, 0.f, 0.f, 1.f
979     };
980
981     memcpy(matrix, m, sizeof(m));
982 }
983
984 static void getZoomMatrix(float zoom, FLOAT matrix[static 16]) {
985
986     const FLOAT m[] = {
987         /* x   y     z     w */
988         1.0f, 0.0f, 0.0f, 0.0f,
989         0.0f, 1.0f, 0.0f, 0.0f,
990         0.0f, 0.0f, 1.0f, 0.0f,
991         0.0f, 0.0f, zoom, 1.0f
992     };
993
994     memcpy(matrix, m, sizeof(m));
995 }
996
997 /* perspective matrix see https://www.opengl.org/sdk/docs/man2/xhtml/gluPerspective.xml */
998 static void getProjectionMatrix(float sar, float fovy, FLOAT matrix[static 16]) {
999
1000     float zFar  = 1000;
1001     float zNear = 0.01;
1002
1003     float f = 1.f / tanf(fovy / 2.f);
1004
1005     const FLOAT m[] = {
1006         f / sar, 0.f,                   0.f,                0.f,
1007         0.f,     f,                     0.f,                0.f,
1008         0.f,     0.f,     (zNear + zFar) / (zNear - zFar), -1.f,
1009         0.f,     0.f, (2 * zNear * zFar) / (zNear - zFar),  0.f};
1010
1011      memcpy(matrix, m, sizeof(m));
1012 }
1013
1014 static float UpdateFOVy(float f_fovx, float f_sar)
1015 {
1016     return 2 * atanf(tanf(f_fovx / 2) / f_sar);
1017 }
1018
1019 static float UpdateZ(float f_fovx, float f_fovy)
1020 {
1021     /* Do trigonometry to calculate the minimal z value
1022      * that will allow us to zoom out without seeing the outside of the
1023      * sphere (black borders). */
1024     float tan_fovx_2 = tanf(f_fovx / 2);
1025     float tan_fovy_2 = tanf(f_fovy / 2);
1026     float z_min = - SPHERE_RADIUS / sinf(atanf(sqrtf(
1027                     tan_fovx_2 * tan_fovx_2 + tan_fovy_2 * tan_fovy_2)));
1028
1029     /* The FOV value above which z is dynamically calculated. */
1030     const float z_thresh = 90.f;
1031
1032     float f_z;
1033     if (f_fovx <= z_thresh * M_PI / 180)
1034         f_z = 0;
1035     else
1036     {
1037         float f = z_min / ((FIELD_OF_VIEW_DEGREES_MAX - z_thresh) * M_PI / 180);
1038         f_z = f * f_fovx - f * z_thresh * M_PI / 180;
1039         if (f_z < z_min)
1040             f_z = z_min;
1041     }
1042     return f_z;
1043 }
1044
1045 static void SetQuadVSProjection(vout_display_t *vd, d3d_quad_t *quad, const vlc_viewpoint_t *p_vp)
1046 {
1047     if (!quad->pVertexShaderConstants)
1048         return;
1049
1050 #define RAD(d) ((float) ((d) * M_PI / 180.f))
1051     float f_fovx = RAD(p_vp->fov);
1052     if ( f_fovx > FIELD_OF_VIEW_DEGREES_MAX * M_PI / 180 + 0.001f ||
1053          f_fovx < -0.001f )
1054         return;
1055
1056     float f_sar = (float) vd->cfg->display.width / vd->cfg->display.height;
1057     float f_teta = RAD(p_vp->yaw) - (float) M_PI_2;
1058     float f_phi  = RAD(p_vp->pitch);
1059     float f_roll = RAD(p_vp->roll);
1060     float f_fovy = UpdateFOVy(f_fovx, f_sar);
1061     float f_z = UpdateZ(f_fovx, f_fovy);
1062
1063     vout_display_sys_t *sys = vd->sys;
1064     HRESULT hr;
1065     D3D11_MAPPED_SUBRESOURCE mapped;
1066     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped);
1067     if (SUCCEEDED(hr)) {
1068         VS_PROJECTION_CONST *dst_data = mapped.pData;
1069         getXRotMatrix(f_phi, dst_data->RotX);
1070         getYRotMatrix(f_teta,   dst_data->RotY);
1071         getZRotMatrix(f_roll,  dst_data->RotZ);
1072         getZoomMatrix(SPHERE_RADIUS * f_z, dst_data->View);
1073         getProjectionMatrix(f_sar, f_fovy, dst_data->Projection);
1074     }
1075     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0);
1076 #undef RAD
1077 }
1078
1079 static void UpdateSize(vout_display_t *vd)
1080 {
1081     vout_display_sys_t *sys = vd->sys;
1082     msg_Dbg(vd, "Detected size change %dx%d", RECTWidth(sys->sys.rect_dest_clipped),
1083             RECTHeight(sys->sys.rect_dest_clipped));
1084
1085     UpdateBackBuffer(vd);
1086
1087     UpdatePicQuadPosition(vd);
1088
1089 #if defined(HAVE_ID3D11VIDEODECODER)
1090     if( sys->context_lock != INVALID_HANDLE_VALUE )
1091     {
1092         WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1093     }
1094 #endif
1095
1096     UpdateQuadPosition(vd, &sys->picQuad, &sys->sys.rect_src_clipped,
1097                        vd->fmt.projection_mode, vd->fmt.orientation);
1098
1099 #if defined(HAVE_ID3D11VIDEODECODER)
1100     if( sys->context_lock != INVALID_HANDLE_VALUE )
1101     {
1102         ReleaseMutex( sys->context_lock );
1103     }
1104 #endif
1105 }
1106
1107 static inline bool RectEquals(const RECT *r1, const RECT *r2)
1108 {
1109     return r1->bottom == r2->bottom && r1->top == r2->top &&
1110            r1->left == r2->left && r1->right == r2->right;
1111 }
1112
1113 #define BEFORE_UPDATE_RECTS \
1114     unsigned int i_outside_width  = vd->fmt.i_width; \
1115     unsigned int i_outside_height = vd->fmt.i_height; \
1116     vd->fmt.i_width  = vd->sys->picQuad.i_width; \
1117     vd->fmt.i_height = vd->sys->picQuad.i_height;
1118 #define AFTER_UPDATE_RECTS \
1119     vd->fmt.i_width  = i_outside_width; \
1120     vd->fmt.i_height = i_outside_height;
1121
1122 static int Control(vout_display_t *vd, int query, va_list args)
1123 {
1124     vout_display_sys_t *sys = vd->sys;
1125     RECT before_src_clipped  = sys->sys.rect_src_clipped;
1126     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
1127     RECT before_dest         = sys->sys.rect_dest;
1128
1129     BEFORE_UPDATE_RECTS;
1130     int res = CommonControl( vd, query, args );
1131     AFTER_UPDATE_RECTS;
1132
1133     if (query == VOUT_DISPLAY_CHANGE_VIEWPOINT)
1134     {
1135         const vout_display_cfg_t *cfg = va_arg(args, const vout_display_cfg_t*);
1136         if ( sys->picQuad.pVertexShaderConstants )
1137         {
1138             SetQuadVSProjection( vd, &sys->picQuad, &cfg->viewpoint );
1139             res = VLC_SUCCESS;
1140         }
1141     }
1142
1143     if (!RectEquals(&before_src_clipped,  &sys->sys.rect_src_clipped) ||
1144         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
1145         !RectEquals(&before_dest,         &sys->sys.rect_dest) )
1146     {
1147         UpdateSize(vd);
1148     }
1149
1150     return res;
1151 }
1152
1153 static void Manage(vout_display_t *vd)
1154 {
1155     vout_display_sys_t *sys = vd->sys;
1156     RECT before_src_clipped  = sys->sys.rect_src_clipped;
1157     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
1158     RECT before_dest         = sys->sys.rect_dest;
1159
1160     BEFORE_UPDATE_RECTS;
1161     CommonManage(vd);
1162     AFTER_UPDATE_RECTS;
1163
1164     if (!RectEquals(&before_src_clipped, &sys->sys.rect_src_clipped) ||
1165         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
1166         !RectEquals(&before_dest, &sys->sys.rect_dest))
1167     {
1168         UpdateSize(vd);
1169     }
1170 }
1171
1172 static void DisplayD3DPicture(vout_display_sys_t *sys, d3d_quad_t *quad, ID3D11ShaderResourceView *resourceView[D3D11_MAX_SHADER_VIEW])
1173 {
1174     UINT stride = sizeof(d3d_vertex_t);
1175     UINT offset = 0;
1176
1177     /* Render the quad */
1178     /* vertex shader */
1179     ID3D11DeviceContext_IASetVertexBuffers(sys->d3dcontext, 0, 1, &quad->pVertexBuffer, &stride, &offset);
1180     ID3D11DeviceContext_IASetIndexBuffer(sys->d3dcontext, quad->pIndexBuffer, DXGI_FORMAT_R16_UINT, 0);
1181     if ( quad->pVertexShaderConstants )
1182         ID3D11DeviceContext_VSSetConstantBuffers(sys->d3dcontext, 0, 1, &quad->pVertexShaderConstants);
1183
1184     ID3D11DeviceContext_VSSetShader(sys->d3dcontext, quad->d3dvertexShader, NULL, 0);
1185
1186     /* pixel shader */
1187     ID3D11DeviceContext_PSSetShader(sys->d3dcontext, quad->d3dpixelShader, NULL, 0);
1188
1189     ID3D11DeviceContext_PSSetConstantBuffers(sys->d3dcontext, 0, quad->PSConstantsCount, quad->pPixelShaderConstants);
1190     ID3D11DeviceContext_PSSetShaderResources(sys->d3dcontext, 0, D3D11_MAX_SHADER_VIEW, resourceView);
1191
1192     ID3D11DeviceContext_RSSetViewports(sys->d3dcontext, 1, &quad->cropViewport);
1193
1194     ID3D11DeviceContext_DrawIndexed(sys->d3dcontext, quad->indexCount, 0, 0);
1195 }
1196
1197 static void Prepare(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
1198 {
1199     vout_display_sys_t *sys = vd->sys;
1200
1201     if (sys->picQuadConfig->formatTexture == DXGI_FORMAT_UNKNOWN)
1202     {
1203         D3D11_MAPPED_SUBRESOURCE mappedResource;
1204         D3D11_TEXTURE2D_DESC texDesc;
1205         int i;
1206         HRESULT hr;
1207
1208         for (i = 0; i < picture->i_planes; i++) {
1209             hr = ID3D11DeviceContext_Map(sys->d3dcontext, sys->stagingSys.resource[i],
1210                                          0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
1211             if( FAILED(hr) )
1212                 break;
1213             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[i], &texDesc);
1214             plane_t texture_plane;
1215             texture_plane.i_lines = texDesc.Height;
1216             texture_plane.i_pitch = mappedResource.RowPitch;
1217             texture_plane.p_pixels = mappedResource.pData;
1218             texture_plane.i_visible_lines = picture->p[i].i_visible_lines;
1219             texture_plane.i_visible_pitch = picture->p[i].i_visible_pitch;
1220             plane_CopyPixels(&texture_plane, &picture->p[i]);
1221             ID3D11DeviceContext_Unmap(sys->d3dcontext, sys->stagingSys.resource[i], 0);
1222         }
1223     }
1224     else
1225     {
1226         picture_sys_t *p_sys = ActivePictureSys(picture);
1227
1228 #if defined(HAVE_ID3D11VIDEODECODER)
1229         if (sys->context_lock != INVALID_HANDLE_VALUE && is_d3d11_opaque(picture->format.i_chroma))
1230             WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1231 #endif
1232         if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader) {
1233             D3D11_TEXTURE2D_DESC texDesc;
1234             if (!is_d3d11_opaque(picture->format.i_chroma))
1235                 Direct3D11UnmapPoolTexture(picture);
1236             ID3D11Texture2D_GetDesc(sys->stagingSys.texture[0], &texDesc);
1237             D3D11_BOX box = {
1238                 .top = 0,
1239                 .bottom = picture->format.i_y_offset + picture->format.i_visible_height,
1240                 .left = 0,
1241                 .right = picture->format.i_x_offset + picture->format.i_visible_width,
1242                 .back = 1,
1243             };
1244             if ( sys->picQuadConfig->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
1245                  sys->picQuadConfig->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
1246             {
1247                 box.bottom = (box.bottom + 0x01) & ~0x01;
1248                 box.right  = (box.right  + 0x01) & ~0x01;
1249             }
1250             assert(box.right <= texDesc.Width);
1251             assert(box.bottom <= texDesc.Height);
1252             ID3D11DeviceContext_CopySubresourceRegion(sys->d3dcontext,
1253                                                       sys->stagingSys.resource[KNOWN_DXGI_INDEX],
1254                                                       0, 0, 0, 0,
1255                                                       p_sys->resource[KNOWN_DXGI_INDEX],
1256                                                       p_sys->slice_index, &box);
1257         }
1258         else
1259         {
1260             D3D11_TEXTURE2D_DESC texDesc;
1261             ID3D11Texture2D_GetDesc(p_sys->texture[0], &texDesc);
1262             if (texDesc.BindFlags & D3D11_BIND_SHADER_RESOURCE)
1263             {
1264                 /* for performance reason we don't want to allocate this during
1265                  * display, do it preferrably when creating the texture */
1266                 assert(p_sys->resourceView[0]!=NULL);
1267             }
1268
1269             if ( sys->picQuad.i_height != texDesc.Height ||
1270                  sys->picQuad.i_width != texDesc.Width )
1271             {
1272                 /* the decoder produced different sizes than the vout, we need to
1273                  * adjust the vertex */
1274                 sys->picQuad.i_height = texDesc.Height;
1275                 sys->picQuad.i_width = texDesc.Width;
1276
1277                 BEFORE_UPDATE_RECTS;
1278                 UpdateRects(vd, NULL, true);
1279                 AFTER_UPDATE_RECTS;
1280                 UpdateSize(vd);
1281             }
1282         }
1283     }
1284
1285     if (subpicture) {
1286         int subpicture_region_count    = 0;
1287         picture_t **subpicture_regions = NULL;
1288         Direct3D11MapSubpicture(vd, &subpicture_region_count, &subpicture_regions, subpicture);
1289         Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
1290         sys->d3dregion_count = subpicture_region_count;
1291         sys->d3dregions      = subpicture_regions;
1292     }
1293
1294     FLOAT blackRGBA[4] = {0.0f, 0.0f, 0.0f, 1.0f};
1295     ID3D11DeviceContext_ClearRenderTargetView(sys->d3dcontext, sys->d3drenderTargetView, blackRGBA);
1296
1297     /* no ID3D11Device operations should come here */
1298
1299     ID3D11DeviceContext_OMSetRenderTargets(sys->d3dcontext, 1, &sys->d3drenderTargetView, sys->d3ddepthStencilView);
1300
1301     ID3D11DeviceContext_ClearDepthStencilView(sys->d3dcontext, sys->d3ddepthStencilView, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);
1302
1303     /* Render the quad */
1304     if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader)
1305         DisplayD3DPicture(sys, &sys->picQuad, sys->stagingSys.resourceView);
1306     else {
1307         picture_sys_t *p_sys = ActivePictureSys(picture);
1308         DisplayD3DPicture(sys, &sys->picQuad, p_sys->resourceView);
1309     }
1310
1311     if (subpicture) {
1312         // draw the additional vertices
1313         for (int i = 0; i < sys->d3dregion_count; ++i) {
1314             if (sys->d3dregions[i])
1315             {
1316                 d3d_quad_t *quad = (d3d_quad_t *) sys->d3dregions[i]->p_sys;
1317                 DisplayD3DPicture(sys, quad, quad->picSys.resourceView);
1318             }
1319         }
1320     }
1321
1322 #if defined(HAVE_ID3D11VIDEODECODER)
1323     if (sys->context_lock != INVALID_HANDLE_VALUE && is_d3d11_opaque(picture->format.i_chroma))
1324     {
1325         ReleaseMutex( sys->context_lock );
1326     }
1327 #endif
1328
1329     if (sys->dxgiswapChain4 && picture->format.mastering.max_luminance)
1330     {
1331         DXGI_HDR_METADATA_HDR10 hdr10 = {0};
1332         hdr10.GreenPrimary[0] = picture->format.mastering.primaries[0];
1333         hdr10.GreenPrimary[1] = picture->format.mastering.primaries[1];
1334         hdr10.BluePrimary[0]  = picture->format.mastering.primaries[2];
1335         hdr10.BluePrimary[1]  = picture->format.mastering.primaries[3];
1336         hdr10.RedPrimary[0]   = picture->format.mastering.primaries[4];
1337         hdr10.RedPrimary[1]   = picture->format.mastering.primaries[5];
1338         hdr10.WhitePoint[0] = picture->format.mastering.white_point[0];
1339         hdr10.WhitePoint[1] = picture->format.mastering.white_point[1];
1340         hdr10.MinMasteringLuminance = picture->format.mastering.min_luminance;
1341         hdr10.MaxMasteringLuminance = picture->format.mastering.max_luminance;
1342         hdr10.MaxContentLightLevel = picture->format.lighting.MaxCLL;
1343         hdr10.MaxFrameAverageLightLevel = picture->format.lighting.MaxFALL;
1344         IDXGISwapChain4_SetHDRMetaData(sys->dxgiswapChain4, DXGI_HDR_METADATA_TYPE_HDR10, sizeof(hdr10), &hdr10);
1345     }
1346
1347     ID3D11DeviceContext_Flush(sys->d3dcontext);
1348 }
1349
1350 static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
1351 {
1352     vout_display_sys_t *sys = vd->sys;
1353
1354     DXGI_PRESENT_PARAMETERS presentParams;
1355     memset(&presentParams, 0, sizeof(presentParams));
1356     HRESULT hr = IDXGISwapChain1_Present1(sys->dxgiswapChain, 0, 0, &presentParams);
1357     if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
1358     {
1359         /* TODO device lost */
1360         msg_Dbg(vd, "SwapChain Present failed. (hr=0x%lX)", hr);
1361     }
1362
1363     picture_Release(picture);
1364     if (subpicture)
1365         subpicture_Delete(subpicture);
1366
1367     CommonDisplay(vd);
1368 }
1369
1370 static void Direct3D11Destroy(vout_display_t *vd)
1371 {
1372 #if !VLC_WINSTORE_APP
1373     vout_display_sys_t *sys = vd->sys;
1374
1375     if (sys->hd3d11_dll)
1376         FreeLibrary(sys->hd3d11_dll);
1377     if (sys->hd3dcompiler_dll)
1378         FreeLibrary(sys->hd3dcompiler_dll);
1379
1380     sys->OurD3D11CreateDevice = NULL;
1381     sys->OurD3D11CreateDeviceAndSwapChain = NULL;
1382     sys->OurD3DCompile = NULL;
1383     sys->hdxgi_dll = NULL;
1384     sys->hd3d11_dll = NULL;
1385     sys->hd3dcompiler_dll = NULL;
1386 #else
1387     VLC_UNUSED(vd);
1388 #endif
1389 }
1390
1391 #if !VLC_WINSTORE_APP
1392 static HINSTANCE Direct3D11LoadShaderLibrary(void)
1393 {
1394     HINSTANCE instance = NULL;
1395     /* d3dcompiler_47 is the latest on windows 8.1 */
1396     for (int i = 47; i > 41; --i) {
1397         TCHAR filename[19];
1398         _sntprintf(filename, 19, TEXT("D3DCOMPILER_%d.dll"), i);
1399         instance = LoadLibrary(filename);
1400         if (instance) break;
1401     }
1402     return instance;
1403 }
1404 #endif
1405
1406 #define COLOR_RANGE_FULL   1 /* 0-255 */
1407 #define COLOR_RANGE_STUDIO 0 /* 16-235 */
1408
1409 #define TRANSFER_FUNC_10    TRANSFER_FUNC_LINEAR
1410 #define TRANSFER_FUNC_22    TRANSFER_FUNC_SRGB
1411 #define TRANSFER_FUNC_2084  TRANSFER_FUNC_SMPTE_ST2084
1412
1413 #define COLOR_PRIMARIES_BT601  COLOR_PRIMARIES_BT601_525
1414
1415 static const dxgi_color_space color_spaces[] = {
1416 #define DXGIMAP(AXIS, RANGE, GAMMA, SITTING, PRIMARIES) \
1417     { DXGI_COLOR_SPACE_##AXIS##_##RANGE##_G##GAMMA##_##SITTING##_P##PRIMARIES, \
1418       #AXIS " Rec." #PRIMARIES " gamma:" #GAMMA " range:" #RANGE, \
1419       COLOR_AXIS_##AXIS, COLOR_PRIMARIES_BT##PRIMARIES, TRANSFER_FUNC_##GAMMA, \
1420       COLOR_SPACE_BT##PRIMARIES, COLOR_RANGE_##RANGE},
1421
1422     DXGIMAP(RGB,   FULL,     22,    NONE,   709)
1423     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   601)
1424     DXGIMAP(YCBCR, FULL,     22,    LEFT,   601)
1425     DXGIMAP(RGB,   FULL,     10,    NONE,   709)
1426     DXGIMAP(RGB,   STUDIO,   22,    NONE,   709)
1427     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   709)
1428     DXGIMAP(YCBCR, FULL,     22,    LEFT,   709)
1429     DXGIMAP(RGB,   STUDIO,   22,    NONE,  2020)
1430     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,  2020)
1431     DXGIMAP(YCBCR, FULL,     22,    LEFT,  2020)
1432     DXGIMAP(YCBCR, STUDIO,   22, TOPLEFT,  2020)
1433     DXGIMAP(RGB,   FULL,     22,    NONE,  2020)
1434     DXGIMAP(RGB,   FULL,   2084,    NONE,  2020)
1435     DXGIMAP(YCBCR, STUDIO, 2084,    LEFT,  2020)
1436     DXGIMAP(RGB,   STUDIO, 2084,    NONE,  2020)
1437     DXGIMAP(YCBCR, STUDIO, 2084, TOPLEFT,  2020)
1438     /*DXGIMAP(YCBCR, FULL,     22,    NONE,  2020, 601)*/
1439     {DXGI_COLOR_SPACE_RESERVED, NULL, 0, 0, 0, 0, 0},
1440 #undef DXGIMAP
1441 };
1442
1443 static void D3D11SetColorSpace(vout_display_t *vd)
1444 {
1445     vout_display_sys_t *sys = vd->sys;
1446     HRESULT hr;
1447     int best = -1;
1448     int score, best_score = 0;
1449     UINT support;
1450     IDXGIOutput *dxgiOutput = NULL;
1451     IDXGISwapChain3 *dxgiswapChain3 = NULL;
1452     sys->display.colorspace = &color_spaces[0];
1453
1454     hr = IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain3, (void **)&dxgiswapChain3);
1455     if (FAILED(hr)) {
1456         msg_Warn(vd, "could not get a IDXGISwapChain3");
1457         goto done;
1458     }
1459
1460     bool src_full_range = vd->source.b_color_range_full ||
1461                           /* the YUV->RGB conversion already output full range */
1462                           is_d3d11_opaque(vd->source.i_chroma) ||
1463                           vlc_fourcc_IsYUV(vd->source.i_chroma);
1464
1465     /* pick the best output based on color support and transfer */
1466     /* TODO support YUV output later */
1467     for (int i=0; color_spaces[i].name; ++i)
1468     {
1469         hr = IDXGISwapChain3_CheckColorSpaceSupport(dxgiswapChain3, color_spaces[i].dxgi, &support);
1470         if (SUCCEEDED(hr) && support) {
1471             msg_Dbg(vd, "supports colorspace %s", color_spaces[i].name);
1472             score = 0;
1473             if (color_spaces[i].primaries == vd->source.primaries)
1474                 score++;
1475             if (color_spaces[i].color == vd->source.space)
1476                 score += 2; /* we don't want to translate color spaces */
1477             if (color_spaces[i].transfer == vd->source.transfer ||
1478                 /* favor 2084 output for HLG source */
1479                 (color_spaces[i].transfer == TRANSFER_FUNC_SMPTE_ST2084 && vd->source.transfer == TRANSFER_FUNC_HLG))
1480                 score++;
1481             if (color_spaces[i].b_full_range == src_full_range)
1482                 score++;
1483             if (score > best_score || (score && best == -1)) {
1484                 best = i;
1485                 best_score = score;
1486             }
1487         }
1488     }
1489
1490     if (best == -1)
1491     {
1492         best = 0;
1493         msg_Warn(vd, "no matching colorspace found force %s", color_spaces[best].name);
1494     }
1495
1496 #ifdef HAVE_DXGI1_6_H
1497     if (SUCCEEDED(IDXGISwapChain_GetContainingOutput( sys->dxgiswapChain, &dxgiOutput )))
1498     {
1499         IDXGIOutput6 *dxgiOutput6 = NULL;
1500         if (SUCCEEDED(IDXGIOutput_QueryInterface( dxgiOutput, &IID_IDXGIOutput6, (void **)&dxgiOutput6 )))
1501         {
1502             DXGI_OUTPUT_DESC1 desc1;
1503             if (SUCCEEDED(IDXGIOutput6_GetDesc1( dxgiOutput6, &desc1 )))
1504             {
1505                 const dxgi_color_space *csp = NULL;
1506                 for (int i=0; color_spaces[i].name; ++i)
1507                 {
1508                     if (color_spaces[i].dxgi == desc1.ColorSpace)
1509                     {
1510                         best = i;
1511                         csp = &color_spaces[i];
1512                         break;
1513                     }
1514                 }
1515
1516                 msg_Dbg(vd, "Output max luminance: %.1f, colorspace %s, bits per pixel %d", desc1.MaxFullFrameLuminance, csp?csp->name:"unknown", desc1.BitsPerColor);
1517                 //sys->display.luminance_peak = desc1.MaxFullFrameLuminance;
1518             }
1519             IDXGIOutput6_Release( dxgiOutput6 );
1520         }
1521         IDXGIOutput_Release( dxgiOutput );
1522     }
1523 #endif
1524
1525     hr = IDXGISwapChain3_SetColorSpace1(dxgiswapChain3, color_spaces[best].dxgi);
1526     if (SUCCEEDED(hr))
1527     {
1528         sys->display.colorspace = &color_spaces[best];
1529         msg_Dbg(vd, "using colorspace %s", sys->display.colorspace->name);
1530     }
1531     else
1532         msg_Err(vd, "Failed to set colorspace %s. (hr=0x%lX)", sys->display.colorspace->name, hr);
1533 done:
1534     /* guestimate the display peak luminance */
1535     switch (sys->display.colorspace->transfer)
1536     {
1537     case TRANSFER_FUNC_LINEAR:
1538     case TRANSFER_FUNC_SRGB:
1539         sys->display.luminance_peak = DEFAULT_SRGB_BRIGHTNESS;
1540         break;
1541     case TRANSFER_FUNC_SMPTE_ST2084:
1542         sys->display.luminance_peak = MAX_PQ_BRIGHTNESS;
1543         break;
1544     /* there is no other output transfer on Windows */
1545     default:
1546         vlc_assert_unreachable();
1547     }
1548
1549     if (dxgiswapChain3)
1550         IDXGISwapChain3_Release(dxgiswapChain3);
1551 }
1552
1553 static const d3d_format_t *GetDirectRenderingFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1554 {
1555     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1556     if (is_d3d11_opaque(i_src_chroma))
1557         supportFlags |= D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1558     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1559 }
1560
1561 static const d3d_format_t *GetDirectDecoderFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1562 {
1563     UINT supportFlags = D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1564     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1565 }
1566
1567 static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor)
1568 {
1569     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1570     if (from_processor)
1571         supportFlags |= D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
1572     return FindD3D11Format( vd->sys->d3ddevice, 0, bit_depth, false, supportFlags );
1573 }
1574
1575 static const d3d_format_t *GetBlendableFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1576 {
1577     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_BLENDABLE;
1578     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, false, supportFlags );
1579 }
1580
1581 static int Direct3D11Open(vout_display_t *vd, video_format_t *fmt)
1582 {
1583     vout_display_sys_t *sys = vd->sys;
1584     IDXGIFactory2 *dxgifactory;
1585
1586 #if !VLC_WINSTORE_APP
1587     HRESULT hr = S_OK;
1588
1589     DXGI_SWAP_CHAIN_DESC1 scd;
1590     memset(&scd, 0, sizeof(scd));
1591     scd.BufferCount = 3;
1592     scd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
1593     scd.SampleDesc.Count = 1;
1594     scd.SampleDesc.Quality = 0;
1595     scd.Width = fmt->i_visible_width;
1596     scd.Height = fmt->i_visible_height;
1597     switch(fmt->i_chroma)
1598     {
1599     case VLC_CODEC_D3D11_OPAQUE_10B:
1600         scd.Format = DXGI_FORMAT_R10G10B10A2_UNORM;
1601         break;
1602     default:
1603         scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM; /* TODO: use DXGI_FORMAT_NV12 */
1604         break;
1605     }
1606     //scd.Flags = 512; // DXGI_SWAP_CHAIN_FLAG_YUV_VIDEO;
1607     scd.SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
1608
1609     hr = D3D11_CreateDevice(VLC_OBJECT(vd), sys->hd3d11_dll,
1610                             is_d3d11_opaque(fmt->i_chroma),
1611                             &sys->d3ddevice, &sys->d3dcontext);
1612     if (FAILED(hr)) {
1613        msg_Err(vd, "Could not Create the D3D11 device. (hr=0x%lX)", hr);
1614        return VLC_EGENERIC;
1615     }
1616
1617     IDXGIAdapter *dxgiadapter = D3D11DeviceAdapter(sys->d3ddevice);
1618     if (unlikely(dxgiadapter==NULL)) {
1619        msg_Err(vd, "Could not get the DXGI Adapter");
1620        return VLC_EGENERIC;
1621     }
1622
1623     hr = IDXGIAdapter_GetParent(dxgiadapter, &IID_IDXGIFactory2, (void **)&dxgifactory);
1624     IDXGIAdapter_Release(dxgiadapter);
1625     if (FAILED(hr)) {
1626        msg_Err(vd, "Could not get the DXGI Factory. (hr=0x%lX)", hr);
1627        return VLC_EGENERIC;
1628     }
1629
1630     hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3ddevice,
1631                                               sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1632     IDXGIFactory2_Release(dxgifactory);
1633     if (FAILED(hr)) {
1634        msg_Err(vd, "Could not create the SwapChain. (hr=0x%lX)", hr);
1635        return VLC_EGENERIC;
1636     }
1637 #endif
1638
1639     IDXGISwapChain_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain4, (void **)&sys->dxgiswapChain4);
1640
1641     D3D11SetColorSpace(vd);
1642
1643     // look for the requested pixel format first
1644     sys->picQuadConfig = GetDirectRenderingFormat(vd, fmt->i_chroma);
1645
1646     /* look for a decoder format that can be decoded but not used in shaders */
1647     const d3d_format_t *decoder_format = NULL;
1648     if ( !sys->picQuadConfig && is_d3d11_opaque(fmt->i_chroma) )
1649         decoder_format = GetDirectDecoderFormat(vd, fmt->i_chroma);
1650     else
1651         decoder_format = sys->picQuadConfig;
1652
1653     // look for any pixel format that we can handle with enough pixels per channel
1654     if ( !sys->picQuadConfig )
1655     {
1656         uint8_t bits_per_channel;
1657         switch (fmt->i_chroma)
1658         {
1659         case VLC_CODEC_D3D11_OPAQUE:
1660             bits_per_channel = 8;
1661             break;
1662         case VLC_CODEC_D3D11_OPAQUE_10B:
1663             bits_per_channel = 10;
1664             break;
1665         default:
1666             {
1667                 const vlc_chroma_description_t *p_format = vlc_fourcc_GetChromaDescription(fmt->i_chroma);
1668                 bits_per_channel = p_format == NULL || p_format->pixel_bits == 0 ? 8 : p_format->pixel_bits;
1669             }
1670             break;
1671         }
1672
1673         sys->picQuadConfig = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL);
1674     }
1675
1676     // look for any pixel format that we can handle
1677     if ( !sys->picQuadConfig )
1678         sys->picQuadConfig = GetDisplayFormatByDepth(vd, 0, false);
1679
1680     if ( !sys->picQuadConfig )
1681     {
1682        msg_Err(vd, "Could not get a suitable texture pixel format");
1683        return VLC_EGENERIC;
1684     }
1685
1686     fmt->i_chroma = decoder_format ? decoder_format->fourcc : sys->picQuadConfig->fourcc;
1687
1688     msg_Dbg( vd, "Using pixel format %s for chroma %4.4s", sys->picQuadConfig->name,
1689                  (char *)&fmt->i_chroma );
1690     DxgiFormatMask( sys->picQuadConfig->formatTexture, fmt );
1691
1692     /* check the region pixel format */
1693     sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_RGBA);
1694     if (!sys->d3dregion_format)
1695         sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_BGRA);
1696
1697     sys->picQuad.i_width  = fmt->i_width;
1698     sys->picQuad.i_height = fmt->i_height;
1699     if (is_d3d11_opaque(fmt->i_chroma))
1700     {
1701         /* worst case scenario we need 128 alignment for HEVC */
1702         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x7F) & ~0x7F;
1703         sys->picQuad.i_height = (sys->picQuad.i_height + 0x7F) & ~0x7F;
1704     }
1705     else if ( sys->picQuadConfig->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
1706               sys->picQuadConfig->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
1707     {
1708         sys->picQuad.i_width  = (sys->picQuad.i_width  + 0x01) & ~0x01;
1709         sys->picQuad.i_height = (sys->picQuad.i_height + 0x01) & ~0x01;
1710     }
1711
1712     BEFORE_UPDATE_RECTS;
1713     UpdateRects(vd, NULL, true);
1714     AFTER_UPDATE_RECTS;
1715
1716     if (Direct3D11CreateResources(vd, fmt)) {
1717         msg_Err(vd, "Failed to allocate resources");
1718         Direct3D11DestroyResources(vd);
1719         return VLC_EGENERIC;
1720     }
1721
1722 #if !VLC_WINSTORE_APP
1723     EventThreadUpdateTitle(sys->sys.event, VOUT_TITLE " (Direct3D11 output)");
1724 #endif
1725
1726     msg_Dbg(vd, "Direct3D11 device adapter successfully initialized");
1727     return VLC_SUCCESS;
1728 }
1729
1730 static void Direct3D11Close(vout_display_t *vd)
1731 {
1732     vout_display_sys_t *sys = vd->sys;
1733
1734     Direct3D11DestroyResources(vd);
1735     if (sys->d3dcontext)
1736     {
1737         ID3D11DeviceContext_Flush(sys->d3dcontext);
1738         ID3D11DeviceContext_Release(sys->d3dcontext);
1739         sys->d3dcontext = NULL;
1740     }
1741     if (sys->d3ddevice)
1742     {
1743         ID3D11Device_Release(sys->d3ddevice);
1744         sys->d3ddevice = NULL;
1745     }
1746     if (sys->dxgiswapChain4)
1747     {
1748         IDXGISwapChain_Release(sys->dxgiswapChain4);
1749         sys->dxgiswapChain4 = NULL;
1750     }
1751     if (sys->dxgiswapChain)
1752     {
1753         IDXGISwapChain_Release(sys->dxgiswapChain);
1754         sys->dxgiswapChain = NULL;
1755     }
1756
1757     msg_Dbg(vd, "Direct3D11 device adapter closed");
1758 }
1759
1760 static void UpdatePicQuadPosition(vout_display_t *vd)
1761 {
1762     vout_display_sys_t *sys = vd->sys;
1763
1764     sys->picQuad.cropViewport.Width    = RECTWidth(sys->sys.rect_dest_clipped);
1765     sys->picQuad.cropViewport.Height   = RECTHeight(sys->sys.rect_dest_clipped);
1766     sys->picQuad.cropViewport.TopLeftX = sys->sys.rect_dest_clipped.left;
1767     sys->picQuad.cropViewport.TopLeftY = sys->sys.rect_dest_clipped.top;
1768
1769     sys->picQuad.cropViewport.MinDepth = 0.0f;
1770     sys->picQuad.cropViewport.MaxDepth = 1.0f;
1771
1772     SetQuadVSProjection(vd, &sys->picQuad, &vd->cfg->viewpoint);
1773
1774 #ifndef NDEBUG
1775     msg_Dbg(vd, "picQuad position (%.02f,%.02f) %.02fx%.02f", sys->picQuad.cropViewport.TopLeftX, sys->picQuad.cropViewport.TopLeftY, sys->picQuad.cropViewport.Width, sys->picQuad.cropViewport.Height );
1776 #endif
1777 }
1778
1779 static ID3DBlob* CompileShader(vout_display_t *vd, const char *psz_shader, bool pixel)
1780 {
1781     vout_display_sys_t *sys = vd->sys;
1782     ID3DBlob* pShaderBlob = NULL, *pErrBlob;
1783
1784     /* TODO : Match the version to the D3D_FEATURE_LEVEL */
1785     HRESULT hr = D3DCompile(psz_shader, strlen(psz_shader),
1786                             NULL, NULL, NULL, "main",
1787                             pixel ? (sys->legacy_shader ? "ps_4_0_level_9_1" : "ps_4_0") :
1788                                     (sys->legacy_shader ? "vs_4_0_level_9_1" : "vs_4_0"),
1789                             0, 0, &pShaderBlob, &pErrBlob);
1790
1791     if (FAILED(hr)) {
1792         char *err = pErrBlob ? ID3D10Blob_GetBufferPointer(pErrBlob) : NULL;
1793         msg_Err(vd, "invalid %s Shader (hr=0x%lX): %s", pixel?"Pixel":"Vertex", hr, err );
1794         if (pErrBlob)
1795             ID3D10Blob_Release(pErrBlob);
1796         return NULL;
1797     }
1798     return pShaderBlob;
1799 }
1800
1801 static bool IsRGBShader(const d3d_format_t *cfg)
1802 {
1803     return cfg->resourceFormat[0] != DXGI_FORMAT_R8_UNORM &&
1804            cfg->resourceFormat[0] != DXGI_FORMAT_R16_UNORM &&
1805            cfg->formatTexture != DXGI_FORMAT_YUY2;
1806 }
1807
1808 static HRESULT CompilePixelShader(vout_display_t *vd, const d3d_format_t *format,
1809                                   video_transfer_func_t transfer, bool src_full_range,
1810                                   ID3D11PixelShader **output)
1811 {
1812     vout_display_sys_t *sys = vd->sys;
1813
1814     static const char *DEFAULT_NOOP = "return rgb";
1815     const char *psz_sampler;
1816     const char *psz_src_transform     = DEFAULT_NOOP;
1817     const char *psz_display_transform = DEFAULT_NOOP;
1818     const char *psz_tone_mapping      = DEFAULT_NOOP;
1819     const char *psz_peak_luminance    = DEFAULT_NOOP;
1820     const char *psz_adjust_range      = DEFAULT_NOOP;
1821     char *psz_luminance = NULL;
1822     char *psz_range = NULL;
1823
1824     switch (format->formatTexture)
1825     {
1826     case DXGI_FORMAT_NV12:
1827     case DXGI_FORMAT_P010:
1828         psz_sampler =
1829                 "sample.x  = shaderTexture[0].Sample(SampleType, In.Texture).x;\
1830                 sample.yz = shaderTexture[1].Sample(SampleType, In.Texture).xy;\
1831                 sample.a  = 1;";
1832         break;
1833     case DXGI_FORMAT_YUY2:
1834         psz_sampler =
1835                 "sample.x  = shaderTexture[0].Sample(SampleType, In.Texture).x;\
1836                 sample.y  = shaderTexture[0].Sample(SampleType, In.Texture).y;\
1837                 sample.z  = shaderTexture[0].Sample(SampleType, In.Texture).a;\
1838                 sample.a  = 1;";
1839         break;
1840     case DXGI_FORMAT_R8G8B8A8_UNORM:
1841     case DXGI_FORMAT_B8G8R8A8_UNORM:
1842     case DXGI_FORMAT_B8G8R8X8_UNORM:
1843     case DXGI_FORMAT_B5G6R5_UNORM:
1844         psz_sampler =
1845                 "sample = shaderTexture[0].Sample(SampleType, In.Texture);";
1846         break;
1847     case DXGI_FORMAT_UNKNOWN:
1848         psz_sampler =
1849                "sample.x  = shaderTexture[0].Sample(SampleType, In.Texture).x;\
1850                 sample.y  = shaderTexture[1].Sample(SampleType, In.Texture).x;\
1851                 sample.z  = shaderTexture[2].Sample(SampleType, In.Texture).x;\
1852                 sample.a  = 1;";
1853         break;
1854     default:
1855         vlc_assert_unreachable();
1856     }
1857
1858     video_transfer_func_t src_transfer;
1859     unsigned src_luminance_peak;
1860     switch (transfer)
1861     {
1862         case TRANSFER_FUNC_SMPTE_ST2084:
1863             if (sys->display.colorspace->transfer == TRANSFER_FUNC_SMPTE_ST2084)
1864                 /* the display will take care of the meta data if there are some */
1865                 src_luminance_peak = MAX_PQ_BRIGHTNESS;
1866             else
1867                 /* TODO adjust this value during playback using HDR metadata ? */
1868                 src_luminance_peak = 5000;
1869             break;
1870         case TRANSFER_FUNC_HLG:
1871             src_luminance_peak = 1000;
1872             break;
1873         case TRANSFER_FUNC_BT470_BG:
1874         case TRANSFER_FUNC_BT470_M:
1875         case TRANSFER_FUNC_BT709:
1876         case TRANSFER_FUNC_SRGB:
1877             src_luminance_peak = DEFAULT_BRIGHTNESS;
1878             break;
1879         default:
1880             msg_Dbg(vd, "unhandled source transfer %d", transfer);
1881             src_luminance_peak = DEFAULT_BRIGHTNESS;
1882             break;
1883     }
1884
1885     if (transfer != sys->display.colorspace->transfer)
1886     {
1887         /* we need to go in linear mode */
1888         switch (transfer)
1889         {
1890             case TRANSFER_FUNC_SMPTE_ST2084:
1891                 /* ST2084 to Linear */
1892                 psz_src_transform =
1893                        ST2084_PQ_CONSTANTS
1894                        "rgb = pow(rgb, 1.0/ST2084_m2);\
1895                         rgb = max(rgb - ST2084_c1, 0.0) / (ST2084_c2 - ST2084_c3 * rgb);\
1896                         rgb = pow(rgb, 1.0/ST2084_m1);\
1897                         return rgb";
1898                 src_transfer = TRANSFER_FUNC_LINEAR;
1899                 break;
1900             case TRANSFER_FUNC_HLG:
1901                 /* HLG to Linear */
1902                 psz_src_transform =
1903                        "rgb.r = inverse_HLG(rgb.r);\
1904                         rgb.g = inverse_HLG(rgb.g);\
1905                         rgb.b = inverse_HLG(rgb.b);\
1906                         return rgb / 20.0";
1907                 src_transfer = TRANSFER_FUNC_LINEAR;
1908                 break;
1909             case TRANSFER_FUNC_BT709:
1910                 psz_src_transform = "return pow(rgb, 1.0 / 0.45)";
1911                 src_transfer = TRANSFER_FUNC_LINEAR;
1912                 break;
1913             case TRANSFER_FUNC_BT470_M:
1914             case TRANSFER_FUNC_SRGB:
1915                 psz_src_transform = "return pow(rgb, 2.2)";
1916                 src_transfer = TRANSFER_FUNC_LINEAR;
1917                 break;
1918             case TRANSFER_FUNC_BT470_BG:
1919                 psz_src_transform = "return pow(rgb, 2.8)";
1920                 src_transfer = TRANSFER_FUNC_LINEAR;
1921                 break;
1922             default:
1923                 msg_Dbg(vd, "unhandled source transfer %d", transfer);
1924                 src_transfer = transfer;
1925                 break;
1926         }
1927
1928         switch (sys->display.colorspace->transfer)
1929         {
1930             case TRANSFER_FUNC_SRGB:
1931                 if (src_transfer == TRANSFER_FUNC_LINEAR)
1932                 {
1933                     /* Linear to sRGB */
1934                     psz_display_transform = "return pow(rgb, 1.0 / 2.2)";
1935
1936                     if (transfer == TRANSFER_FUNC_SMPTE_ST2084 || transfer == TRANSFER_FUNC_HLG)
1937                     {
1938                         /* HDR tone mapping */
1939                         psz_tone_mapping =
1940                             "static const float3 HABLE_DIV = hable(11.2);\
1941                             rgb = hable(rgb) / HABLE_DIV;\
1942                             return rgb";
1943                     }
1944                 }
1945                 else
1946                     msg_Warn(vd, "don't know how to transfer from %d to sRGB", src_transfer);
1947                 break;
1948
1949             case TRANSFER_FUNC_SMPTE_ST2084:
1950                 if (src_transfer == TRANSFER_FUNC_LINEAR)
1951                 {
1952                     /* Linear to ST2084 */
1953                     psz_display_transform =
1954                            ST2084_PQ_CONSTANTS
1955                            "rgb = pow(rgb, ST2084_m1);\
1956                             rgb = (ST2084_c1 + ST2084_c2 * rgb) / (1 + ST2084_c3 * rgb);\
1957                             rgb = pow(rgb, ST2084_m2);\
1958                             return rgb";
1959                 }
1960                 else
1961                     msg_Warn(vd, "don't know how to transfer from %d to SMPTE ST 2084", src_transfer);
1962                 break;
1963             default:
1964                 msg_Warn(vd, "don't know how to transfer from %d to %d", src_transfer, sys->display.colorspace->transfer);
1965                 break;
1966         }
1967     }
1968
1969     if (src_luminance_peak != sys->display.luminance_peak)
1970     {
1971         psz_luminance = malloc(64);
1972         if (likely(psz_luminance))
1973         {
1974             sprintf(psz_luminance, "return rgb * %f", (float)src_luminance_peak / (float)sys->display.luminance_peak);
1975             psz_peak_luminance = psz_luminance;
1976         }
1977     }
1978
1979     int range_adjust = sys->display.colorspace->b_full_range;
1980     if (!IsRGBShader(format))
1981         range_adjust--; /* the YUV->RGB conversion already output full range */
1982     if (src_full_range)
1983         range_adjust--;
1984
1985     if (range_adjust != 0)
1986     {
1987         psz_range = malloc(256);
1988         if (likely(psz_range))
1989         {
1990             FLOAT itu_black_level;
1991             FLOAT itu_range_factor;
1992             FLOAT itu_white_level;
1993             switch (format->bitsPerChannel)
1994             {
1995             case 8:
1996                 /* Rec. ITU-R BT.709-6 §4.6 */
1997                 itu_black_level  =              16.f / 255.f;
1998                 itu_white_level  =             235.f / 255.f;
1999                 itu_range_factor = (float)(235 - 16) / 255.f;
2000                 break;
2001             case 10:
2002                 /* Rec. ITU-R BT.709-6 §4.6 */
2003                 itu_black_level  =              64.f / 1023.f;
2004                 itu_white_level  =             940.f / 1023.f;
2005                 itu_range_factor = (float)(940 - 64) / 1023.f;
2006                 break;
2007             case 12:
2008                 /* Rec. ITU-R BT.2020-2 Table 5 */
2009                 itu_black_level  =               256.f / 4095.f;
2010                 itu_white_level  =              3760.f / 4095.f;
2011                 itu_range_factor = (float)(3760 - 256) / 4095.f;
2012                 break;
2013             default:
2014                 /* unknown bitdepth, use approximation for infinite bit depth */
2015                 itu_black_level  =              16.f / 256.f;
2016                 itu_white_level  =             235.f / 256.f;
2017                 itu_range_factor = (float)(235 - 16) / 256.f;
2018                 break;
2019             }
2020
2021             FLOAT black_level = 0;
2022             FLOAT range_factor = 1.0f;
2023             if (range_adjust > 0)
2024             {
2025                 /* expand the range from studio to full range */
2026                 while (range_adjust--)
2027                 {
2028                     black_level -= itu_black_level;
2029                     range_factor /= itu_range_factor;
2030                 }
2031                 sprintf(psz_range, "return max(0,min(1,(rgb + %f) * %f))",
2032                         black_level, range_factor);
2033             }
2034             else
2035             {
2036                 /* shrink the range to studio range */
2037                 while (range_adjust++)
2038                 {
2039                     black_level += itu_black_level;
2040                     range_factor *= itu_range_factor;
2041                 }
2042                 sprintf(psz_range, "return clamp(rgb + %f * %f,%f,%f)",
2043                         black_level, range_factor, itu_black_level, itu_white_level);
2044             }
2045             psz_adjust_range = psz_range;
2046         }
2047     }
2048
2049     char *shader = malloc(strlen(globPixelShaderDefault) + 32 + strlen(psz_sampler) +
2050                           strlen(psz_src_transform) + strlen(psz_display_transform) +
2051                           strlen(psz_tone_mapping) + strlen(psz_peak_luminance) + strlen(psz_adjust_range));
2052     if (!shader)
2053     {
2054         msg_Err(vd, "no room for the Pixel Shader");
2055         free(psz_luminance);
2056         free(psz_range);
2057         return E_OUTOFMEMORY;
2058     }
2059     sprintf(shader, globPixelShaderDefault, sys->legacy_shader ? "" : "Array", psz_src_transform,
2060             psz_display_transform, psz_tone_mapping, psz_peak_luminance, psz_adjust_range, psz_sampler);
2061 #ifndef NDEBUG
2062     if (!IsRGBShader(format)) {
2063         msg_Dbg(vd,"psz_src_transform %s", psz_src_transform);
2064         msg_Dbg(vd,"psz_tone_mapping %s", psz_tone_mapping);
2065         msg_Dbg(vd,"psz_peak_luminance %s", psz_peak_luminance);
2066         msg_Dbg(vd,"psz_display_transform %s", psz_display_transform);
2067         msg_Dbg(vd,"psz_adjust_range %s", psz_adjust_range);
2068     }
2069 #endif
2070     free(psz_luminance);
2071     free(psz_range);
2072
2073     ID3DBlob *pPSBlob = CompileShader(vd, shader, true);
2074     free(shader);
2075     if (!pPSBlob)
2076         return E_INVALIDARG;
2077
2078     HRESULT hr = ID3D11Device_CreatePixelShader(sys->d3ddevice,
2079                                                 (void *)ID3D10Blob_GetBufferPointer(pPSBlob),
2080                                                 ID3D10Blob_GetBufferSize(pPSBlob), NULL, output);
2081
2082     ID3D10Blob_Release(pPSBlob);
2083     return hr;
2084 }
2085
2086 #if (!VLC_WINSTORE_APP) && defined (HAVE_ID3D11VIDEODECODER)
2087
2088 static HKEY GetAdapterRegistry(DXGI_ADAPTER_DESC *adapterDesc)
2089 {
2090     HKEY hKey;
2091     TCHAR key[128];
2092     TCHAR szData[256], lookup[256];
2093     DWORD len = 256;
2094
2095     _sntprintf(lookup, 256, TEXT("pci\\ven_%04x&dev_%04x&rev_cf"), adapterDesc->VendorId, adapterDesc->DeviceId);
2096     for (int i=0;;i++)
2097     {
2098         _sntprintf(key, 128, TEXT("SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\%04d"), i);
2099         if( RegOpenKeyEx(HKEY_LOCAL_MACHINE, key, 0, KEY_READ, &hKey) != ERROR_SUCCESS )
2100             return NULL;
2101
2102         len = sizeof(szData);
2103         if( RegQueryValueEx( hKey, TEXT("MatchingDeviceId"), NULL, NULL, (LPBYTE) &szData, &len ) == ERROR_SUCCESS ) {
2104             if (_tcscmp(lookup, szData) == 0)
2105                 return hKey;
2106         }
2107
2108         RegCloseKey(hKey);
2109     }
2110     return NULL;
2111 }
2112 #endif
2113
2114 static bool CanUseTextureArray(vout_display_t *vd)
2115 {
2116 #ifndef HAVE_ID3D11VIDEODECODER
2117     (void) vd;
2118     return false;
2119 #else
2120     vout_display_sys_t *sys = vd->sys;
2121     TCHAR szData[256];
2122     DWORD len = 256;
2123     IDXGIAdapter *pAdapter = D3D11DeviceAdapter(sys->d3ddevice);
2124     if (!pAdapter)
2125         return false;
2126
2127     DXGI_ADAPTER_DESC adapterDesc;
2128     HRESULT hr = IDXGIAdapter_GetDesc(pAdapter, &adapterDesc);
2129     IDXGIAdapter_Release(pAdapter);
2130     if (FAILED(hr))
2131         return false;
2132
2133     /* ATI/AMD hardware has issues with pixel shaders with Texture2DArray */
2134     if (adapterDesc.VendorId != 0x1002)
2135         return true;
2136
2137 #if !VLC_WINSTORE_APP
2138     HKEY hKey = GetAdapterRegistry(&adapterDesc);
2139     if (hKey == NULL)
2140         return false;
2141
2142     LONG err = RegQueryValueEx( hKey, TEXT("DriverVersion"), NULL, NULL, (LPBYTE) &szData, &len );
2143     RegCloseKey(hKey);
2144
2145     if (err == ERROR_SUCCESS )
2146     {
2147         int wddm, d3d_features, revision, build;
2148         /* see https://msdn.microsoft.com/windows/hardware/commercialize/design/compatibility/device-graphics */
2149         if (_stscanf(szData, TEXT("%d.%d.%d.%d"), &wddm, &d3d_features, &revision, &build) == 4)
2150             /* it should be OK starting from driver 22.19.162.xxx */
2151             return wddm > 22 || (wddm == 22 && (d3d_features > 19 || (d3d_features == 19 && revision >= 162)));
2152     }
2153 #endif
2154     msg_Dbg(vd, "fallback to legacy shader mode for old AMD drivers");
2155     return false;
2156 #endif
2157 }
2158
2159 /* TODO : handle errors better
2160    TODO : seperate out into smaller functions like createshaders */
2161 static int Direct3D11CreateResources(vout_display_t *vd, video_format_t *fmt)
2162 {
2163     vout_display_sys_t *sys = vd->sys;
2164     HRESULT hr;
2165
2166 #if defined(HAVE_ID3D11VIDEODECODER)
2167     sys->context_lock = CreateMutexEx( NULL, NULL, 0, SYNCHRONIZE );
2168     ID3D11Device_SetPrivateData( sys->d3ddevice, &GUID_CONTEXT_MUTEX, sizeof( sys->context_lock ), &sys->context_lock );
2169 #endif
2170
2171     hr = UpdateBackBuffer(vd);
2172     if (FAILED(hr)) {
2173        msg_Err(vd, "Could not update the backbuffer. (hr=0x%lX)", hr);
2174        return VLC_EGENERIC;
2175     }
2176
2177     ID3D11BlendState *pSpuBlendState;
2178     D3D11_BLEND_DESC spuBlendDesc = { 0 };
2179     spuBlendDesc.RenderTarget[0].BlendEnable = TRUE;
2180     spuBlendDesc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA;
2181     spuBlendDesc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA;
2182     spuBlendDesc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD;
2183
2184     spuBlendDesc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE;
2185     spuBlendDesc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO;
2186     spuBlendDesc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;
2187
2188     spuBlendDesc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
2189
2190     spuBlendDesc.RenderTarget[1].BlendEnable = TRUE;
2191     spuBlendDesc.RenderTarget[1].SrcBlend = D3D11_BLEND_ONE;
2192     spuBlendDesc.RenderTarget[1].DestBlend = D3D11_BLEND_ZERO;
2193     spuBlendDesc.RenderTarget[1].BlendOp = D3D11_BLEND_OP_ADD;
2194
2195     spuBlendDesc.RenderTarget[1].SrcBlendAlpha = D3D11_BLEND_ONE;
2196     spuBlendDesc.RenderTarget[1].DestBlendAlpha = D3D11_BLEND_ZERO;
2197     spuBlendDesc.RenderTarget[1].BlendOpAlpha = D3D11_BLEND_OP_ADD;
2198
2199     spuBlendDesc.RenderTarget[1].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
2200     hr = ID3D11Device_CreateBlendState(sys->d3ddevice, &spuBlendDesc, &pSpuBlendState);
2201     if (FAILED(hr)) {
2202        msg_Err(vd, "Could not create SPU blend state. (hr=0x%lX)", hr);
2203        return VLC_EGENERIC;
2204     }
2205     ID3D11DeviceContext_OMSetBlendState(sys->d3dcontext, pSpuBlendState, NULL, 0xFFFFFFFF);
2206     ID3D11BlendState_Release(pSpuBlendState);
2207
2208     /* disable depth testing as we're only doing 2D
2209      * see https://msdn.microsoft.com/en-us/library/windows/desktop/bb205074%28v=vs.85%29.aspx
2210      * see http://rastertek.com/dx11tut11.html
2211     */
2212     D3D11_DEPTH_STENCIL_DESC stencilDesc;
2213     ZeroMemory(&stencilDesc, sizeof(stencilDesc));
2214
2215     ID3D11DepthStencilState *pDepthStencilState;
2216     hr = ID3D11Device_CreateDepthStencilState(sys->d3ddevice, &stencilDesc, &pDepthStencilState );
2217     if (SUCCEEDED(hr)) {
2218         ID3D11DeviceContext_OMSetDepthStencilState(sys->d3dcontext, pDepthStencilState, 0);
2219         ID3D11DepthStencilState_Release(pDepthStencilState);
2220     }
2221
2222     sys->legacy_shader = !CanUseTextureArray(vd);
2223     vd->info.is_slow = !is_d3d11_opaque(fmt->i_chroma) && sys->picQuadConfig->formatTexture != DXGI_FORMAT_UNKNOWN;
2224
2225     hr = CompilePixelShader(vd, sys->picQuadConfig, fmt->transfer, fmt->b_color_range_full, &sys->picQuadPixelShader);
2226     if (FAILED(hr))
2227     {
2228 #ifdef HAVE_ID3D11VIDEODECODER
2229         if (!sys->legacy_shader)
2230         {
2231             sys->legacy_shader = true;
2232             msg_Dbg(vd, "fallback to legacy shader mode");
2233             hr = CompilePixelShader(vd, sys->picQuadConfig, fmt->transfer, fmt->b_color_range_full, &sys->picQuadPixelShader);
2234         }
2235 #endif
2236         if (FAILED(hr))
2237         {
2238             msg_Err(vd, "Failed to create the pixel shader. (hr=0x%lX)", hr);
2239             return VLC_EGENERIC;
2240         }
2241     }
2242
2243     if (sys->d3dregion_format != NULL)
2244     {
2245         hr = CompilePixelShader(vd, sys->d3dregion_format, TRANSFER_FUNC_SRGB, true, &sys->pSPUPixelShader);
2246         if (FAILED(hr))
2247         {
2248             ID3D11PixelShader_Release(sys->picQuadPixelShader);
2249             sys->picQuadPixelShader = NULL;
2250             msg_Err(vd, "Failed to create the SPU pixel shader. (hr=0x%lX)", hr);
2251             return VLC_EGENERIC;
2252         }
2253     }
2254
2255     ID3DBlob *pVSBlob = CompileShader(vd, globVertexShaderFlat , false);
2256     if (!pVSBlob)
2257         return VLC_EGENERIC;
2258
2259     hr = ID3D11Device_CreateVertexShader(sys->d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2260                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->flatVSShader);
2261
2262     if(FAILED(hr)) {
2263       ID3D11Device_Release(pVSBlob);
2264       msg_Err(vd, "Failed to create the flat vertex shader. (hr=0x%lX)", hr);
2265       return VLC_EGENERIC;
2266     }
2267
2268     D3D11_INPUT_ELEMENT_DESC layout[] = {
2269     { "POSITION", 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
2270     { "TEXCOORD", 0, DXGI_FORMAT_R32G32_FLOAT,    0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
2271     };
2272
2273     ID3D11InputLayout* pVertexLayout = NULL;
2274     hr = ID3D11Device_CreateInputLayout(sys->d3ddevice, layout, 2, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2275                                         ID3D10Blob_GetBufferSize(pVSBlob), &pVertexLayout);
2276
2277     ID3D10Blob_Release(pVSBlob);
2278
2279     if(FAILED(hr)) {
2280       msg_Err(vd, "Failed to create the vertex input layout. (hr=0x%lX)", hr);
2281       return VLC_EGENERIC;
2282     }
2283     ID3D11DeviceContext_IASetInputLayout(sys->d3dcontext, pVertexLayout);
2284     ID3D11InputLayout_Release(pVertexLayout);
2285
2286     pVSBlob = CompileShader(vd, globVertexShaderProjection, false);
2287     if (!pVSBlob)
2288         return VLC_EGENERIC;
2289
2290     hr = ID3D11Device_CreateVertexShader(sys->d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2291                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->projectionVSShader);
2292
2293     if(FAILED(hr)) {
2294       ID3D11Device_Release(pVSBlob);
2295       msg_Err(vd, "Failed to create the projection vertex shader. (hr=0x%lX)", hr);
2296       return VLC_EGENERIC;
2297     }
2298     ID3D10Blob_Release(pVSBlob);
2299
2300     ID3D11DeviceContext_IASetPrimitiveTopology(sys->d3dcontext, D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
2301
2302     UpdatePicQuadPosition(vd);
2303
2304     D3D11_SAMPLER_DESC sampDesc;
2305     memset(&sampDesc, 0, sizeof(sampDesc));
2306     sampDesc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT;
2307     sampDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP;
2308     sampDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP;
2309     sampDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP;
2310     sampDesc.ComparisonFunc = D3D11_COMPARISON_ALWAYS;
2311     sampDesc.MinLOD = 0;
2312     sampDesc.MaxLOD = D3D11_FLOAT32_MAX;
2313
2314     ID3D11SamplerState *d3dsampState;
2315     hr = ID3D11Device_CreateSamplerState(sys->d3ddevice, &sampDesc, &d3dsampState);
2316
2317     if (FAILED(hr)) {
2318       msg_Err(vd, "Could not Create the D3d11 Sampler State. (hr=0x%lX)", hr);
2319       return VLC_EGENERIC;
2320     }
2321     ID3D11DeviceContext_PSSetSamplers(sys->d3dcontext, 0, 1, &d3dsampState);
2322     ID3D11SamplerState_Release(d3dsampState);
2323
2324     msg_Dbg(vd, "Direct3D11 resources created");
2325     return VLC_SUCCESS;
2326 }
2327
2328 static void Direct3D11DestroyPool(vout_display_t *vd)
2329 {
2330     vout_display_sys_t *sys = vd->sys;
2331
2332     if (sys->sys.pool)
2333         picture_pool_Release(sys->sys.pool);
2334     sys->sys.pool = NULL;
2335 }
2336
2337 /**
2338  * Compute the vertex ordering needed to rotate the video. Without
2339  * rotation, the vertices of the rectangle are defined in a clockwise
2340  * order. This function computes a remapping of the coordinates to
2341  * implement the rotation, given fixed texture coordinates.
2342  * The unrotated order is the following:
2343  * 0--1
2344  * |  |
2345  * 3--2
2346  * For a 180 degrees rotation it should like this:
2347  * 2--3
2348  * |  |
2349  * 1--0
2350  * Vertex 0 should be assigned coordinates at index 2 from the
2351  * unrotated order and so on, thus yielding order: 2 3 0 1.
2352  */
2353 static void orientationVertexOrder(video_orientation_t orientation, int vertex_order[static 4])
2354 {
2355     switch (orientation) {
2356         case ORIENT_ROTATED_90:
2357             vertex_order[0] = 3;
2358             vertex_order[1] = 0;
2359             vertex_order[2] = 1;
2360             vertex_order[3] = 2;
2361             break;
2362         case ORIENT_ROTATED_270:
2363             vertex_order[0] = 1;
2364             vertex_order[1] = 2;
2365             vertex_order[2] = 3;
2366             vertex_order[3] = 0;
2367             break;
2368         case ORIENT_ROTATED_180:
2369             vertex_order[0] = 2;
2370             vertex_order[1] = 3;
2371             vertex_order[2] = 0;
2372             vertex_order[3] = 1;
2373             break;
2374         case ORIENT_TRANSPOSED:
2375             vertex_order[0] = 2;
2376             vertex_order[1] = 1;
2377             vertex_order[2] = 0;
2378             vertex_order[3] = 3;
2379             break;
2380         case ORIENT_HFLIPPED:
2381             vertex_order[0] = 3;
2382             vertex_order[1] = 2;
2383             vertex_order[2] = 1;
2384             vertex_order[3] = 0;
2385             break;
2386         case ORIENT_VFLIPPED:
2387             vertex_order[0] = 1;
2388             vertex_order[1] = 0;
2389             vertex_order[2] = 3;
2390             vertex_order[3] = 2;
2391             break;
2392         case ORIENT_ANTI_TRANSPOSED: /* transpose + vflip */
2393             vertex_order[0] = 1;
2394             vertex_order[1] = 2;
2395             vertex_order[2] = 3;
2396             vertex_order[3] = 0;
2397             break;
2398        default:
2399             vertex_order[0] = 0;
2400             vertex_order[1] = 1;
2401             vertex_order[2] = 2;
2402             vertex_order[3] = 3;
2403             break;
2404     }
2405 }
2406
2407 static void SetupQuadFlat(d3d_vertex_t *dst_data, const RECT *output,
2408                           const d3d_quad_t *quad,
2409                           WORD *triangle_pos, video_orientation_t orientation)
2410 {
2411     unsigned int dst_x = output->left;
2412     unsigned int dst_y = output->top;
2413     unsigned int dst_width = RECTWidth(*output);
2414     unsigned int dst_height = RECTHeight(*output);
2415     unsigned int src_width = quad->i_width;
2416     unsigned int src_height = quad->i_height;
2417     LONG MidY = output->top + output->bottom; // /2
2418     LONG MidX = output->left + output->right; // /2
2419
2420     /* the clamping will not work properly on the side of the texture as it
2421      * will have decoder pixels not mean to be displayed but used for interpolation
2422      * So we lose the last line that will be partially green */
2423     if (src_width != dst_width)
2424         src_width += 1;
2425     if (src_height != dst_height)
2426         src_height += 1;
2427
2428     float top, bottom, left, right;
2429     top    =  (float)MidY / (float)(MidY - 2*dst_y);
2430     bottom = -(2.f*src_height - MidY) / (float)(MidY - 2*dst_y);
2431     right  =  (2.f*src_width  - MidX) / (float)(MidX - 2*dst_x);
2432     left   = -(float)MidX / (float)(MidX - 2*dst_x);
2433
2434     const float vertices_coords[4][2] = {
2435         { left,  bottom },
2436         { right, bottom },
2437         { right, top    },
2438         { left,  top    },
2439     };
2440
2441     /* Compute index remapping necessary to implement the rotation. */
2442     int vertex_order[4];
2443     orientationVertexOrder(orientation, vertex_order);
2444
2445     for (int i = 0; i < 4; ++i) {
2446         dst_data[i].position.x  = vertices_coords[vertex_order[i]][0];
2447         dst_data[i].position.y  = vertices_coords[vertex_order[i]][1];
2448     }
2449
2450     // bottom left
2451     dst_data[0].position.z = 0.0f;
2452     dst_data[0].texture.x = 0.0f;
2453     dst_data[0].texture.y = 1.0f;
2454
2455     // bottom right
2456     dst_data[1].position.z = 0.0f;
2457     dst_data[1].texture.x = 1.0f;
2458     dst_data[1].texture.y = 1.0f;
2459
2460     // top right
2461     dst_data[2].position.z = 0.0f;
2462     dst_data[2].texture.x = 1.0f;
2463     dst_data[2].texture.y = 0.0f;
2464
2465     // top left
2466     dst_data[3].position.z = 0.0f;
2467     dst_data[3].texture.x = 0.0f;
2468     dst_data[3].texture.y = 0.0f;
2469
2470     triangle_pos[0] = 3;
2471     triangle_pos[1] = 1;
2472     triangle_pos[2] = 0;
2473
2474     triangle_pos[3] = 2;
2475     triangle_pos[4] = 1;
2476     triangle_pos[5] = 3;
2477 }
2478
2479 #define SPHERE_SLICES 128
2480 #define nbLatBands SPHERE_SLICES
2481 #define nbLonBands SPHERE_SLICES
2482
2483 static void SetupQuadSphere(d3d_vertex_t *dst_data, WORD *triangle_pos)
2484 {
2485     for (unsigned lat = 0; lat <= nbLatBands; lat++) {
2486         float theta = lat * (float) M_PI / nbLatBands;
2487         float sinTheta, cosTheta;
2488
2489         sincosf(theta, &sinTheta, &cosTheta);
2490
2491         for (unsigned lon = 0; lon <= nbLonBands; lon++) {
2492             float phi = lon * 2 * (float) M_PI / nbLonBands;
2493             float sinPhi, cosPhi;
2494
2495             sincosf(phi, &sinPhi, &cosPhi);
2496
2497             float x = cosPhi * sinTheta;
2498             float y = cosTheta;
2499             float z = sinPhi * sinTheta;
2500
2501             unsigned off1 = lat * (nbLonBands + 1) + lon;
2502             dst_data[off1].position.x = SPHERE_RADIUS * x;
2503             dst_data[off1].position.y = SPHERE_RADIUS * y;
2504             dst_data[off1].position.z = SPHERE_RADIUS * z;
2505
2506             dst_data[off1].texture.x = lon / (float) nbLonBands; // 0(left) to 1(right)
2507             dst_data[off1].texture.y = lat / (float) nbLatBands; // 0(top) to 1 (bottom)
2508         }
2509     }
2510
2511     for (unsigned lat = 0; lat < nbLatBands; lat++) {
2512         for (unsigned lon = 0; lon < nbLonBands; lon++) {
2513             unsigned first = (lat * (nbLonBands + 1)) + lon;
2514             unsigned second = first + nbLonBands + 1;
2515
2516             unsigned off = (lat * nbLatBands + lon) * 3 * 2;
2517
2518             triangle_pos[off] = first;
2519             triangle_pos[off + 1] = first + 1;
2520             triangle_pos[off + 2] = second;
2521
2522             triangle_pos[off + 3] = second;
2523             triangle_pos[off + 4] = first + 1;
2524             triangle_pos[off + 5] = second + 1;
2525         }
2526     }
2527 }
2528
2529 static bool AllocQuadVertices(vout_display_t *vd, d3d_quad_t *quad,
2530                               video_projection_mode_t projection)
2531 {
2532     HRESULT hr;
2533     vout_display_sys_t *sys = vd->sys;
2534
2535     if (projection == PROJECTION_MODE_RECTANGULAR)
2536     {
2537         quad->vertexCount = 4;
2538         quad->indexCount = 2 * 3;
2539     }
2540     else if (projection == PROJECTION_MODE_EQUIRECTANGULAR)
2541     {
2542         quad->vertexCount = (SPHERE_SLICES+1) * (SPHERE_SLICES+1);
2543         quad->indexCount = nbLatBands * nbLonBands * 2 * 3;
2544     }
2545     else
2546     {
2547         msg_Warn(vd, "Projection mode %d not handled", projection);
2548         return false;
2549     }
2550
2551     D3D11_BUFFER_DESC bd;
2552     memset(&bd, 0, sizeof(bd));
2553     bd.Usage = D3D11_USAGE_DYNAMIC;
2554     bd.ByteWidth = sizeof(d3d_vertex_t) * quad->vertexCount;
2555     bd.BindFlags = D3D11_BIND_VERTEX_BUFFER;
2556     bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
2557
2558     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &bd, NULL, &quad->pVertexBuffer);
2559     if(FAILED(hr)) {
2560       msg_Err(vd, "Failed to create vertex buffer. (hr=%lX)", hr);
2561       return false;
2562     }
2563
2564     /* create the index of the vertices */
2565     D3D11_BUFFER_DESC quadDesc = {
2566         .Usage = D3D11_USAGE_DYNAMIC,
2567         .ByteWidth = sizeof(WORD) * quad->indexCount,
2568         .BindFlags = D3D11_BIND_INDEX_BUFFER,
2569         .CPUAccessFlags = D3D11_CPU_ACCESS_WRITE,
2570     };
2571
2572     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &quadDesc, NULL, &quad->pIndexBuffer);
2573     if(FAILED(hr)) {
2574         msg_Err(vd, "Could not create the quad indices. (hr=0x%lX)", hr);
2575         return false;
2576     }
2577
2578     return true;
2579 }
2580
2581 static bool UpdateQuadPosition( vout_display_t *vd, d3d_quad_t *quad,
2582                                 const RECT *output,
2583                                 video_projection_mode_t projection,
2584                                 video_orientation_t orientation )
2585 {
2586     vout_display_sys_t *sys = vd->sys;
2587     HRESULT hr;
2588     D3D11_MAPPED_SUBRESOURCE mappedResource;
2589
2590     /* create the vertices */
2591     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2592     if (FAILED(hr)) {
2593         msg_Err(vd, "Failed to lock the vertex buffer (hr=0x%lX)", hr);
2594         return false;
2595     }
2596     d3d_vertex_t *dst_data = mappedResource.pData;
2597
2598     /* create the vertex indices */
2599     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pIndexBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2600     if (FAILED(hr)) {
2601         msg_Err(vd, "Failed to lock the index buffer (hr=0x%lX)", hr);
2602         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0);
2603         return false;
2604     }
2605     WORD *triangle_pos = mappedResource.pData;
2606
2607     if ( projection == PROJECTION_MODE_RECTANGULAR )
2608         SetupQuadFlat(dst_data, output, quad, triangle_pos, orientation);
2609     else
2610         SetupQuadSphere(dst_data, triangle_pos);
2611
2612     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pIndexBuffer, 0);
2613     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0);
2614
2615     return true;
2616 }
2617
2618 static int SetupQuad(vout_display_t *vd, const video_format_t *fmt, d3d_quad_t *quad,
2619                      const RECT *output,
2620                      const d3d_format_t *cfg, ID3D11PixelShader *d3dpixelShader,
2621                      video_projection_mode_t projection, video_orientation_t orientation)
2622 {
2623     vout_display_sys_t *sys = vd->sys;
2624     HRESULT hr;
2625     const bool RGB_shader = IsRGBShader(cfg);
2626
2627     /* pixel shader constant buffer */
2628     PS_CONSTANT_BUFFER defaultConstants = {
2629       .Opacity = 1,
2630     };
2631     static_assert((sizeof(PS_CONSTANT_BUFFER)%16)==0,"Constant buffers require 16-byte alignment");
2632     D3D11_BUFFER_DESC constantDesc = {
2633         .Usage = D3D11_USAGE_DYNAMIC,
2634         .ByteWidth = sizeof(PS_CONSTANT_BUFFER),
2635         .BindFlags = D3D11_BIND_CONSTANT_BUFFER,
2636         .CPUAccessFlags = D3D11_CPU_ACCESS_WRITE,
2637     };
2638     D3D11_SUBRESOURCE_DATA constantInit = { .pSysMem = &defaultConstants };
2639     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, &constantInit, &quad->pPixelShaderConstants[0]);
2640     if(FAILED(hr)) {
2641         msg_Err(vd, "Could not create the pixel shader constant buffer. (hr=0x%lX)", hr);
2642         goto error;
2643     }
2644
2645     FLOAT itu_black_level = 0.f;
2646     FLOAT itu_achromacy   = 0.f;
2647     if (!RGB_shader)
2648     {
2649         switch (cfg->bitsPerChannel)
2650         {
2651         case 8:
2652             /* Rec. ITU-R BT.709-6 §4.6 */
2653             itu_black_level  =              16.f / 255.f;
2654             itu_achromacy    =             128.f / 255.f;
2655             break;
2656         case 10:
2657             /* Rec. ITU-R BT.709-6 §4.6 */
2658             itu_black_level  =              64.f / 1023.f;
2659             itu_achromacy    =             512.f / 1023.f;
2660             break;
2661         case 12:
2662             /* Rec. ITU-R BT.2020-2 Table 5 */
2663             itu_black_level  =               256.f / 4095.f;
2664             itu_achromacy    =              2048.f / 4095.f;
2665             break;
2666         default:
2667             /* unknown bitdepth, use approximation for infinite bit depth */
2668             itu_black_level  =              16.f / 256.f;
2669             itu_achromacy    =             128.f / 256.f;
2670             break;
2671         }
2672     }
2673
2674     static const FLOAT IDENTITY_4X4[4 * 4] = {
2675         1.f, 0.f, 0.f, 0.f,
2676         0.f, 1.f, 0.f, 0.f,
2677         0.f, 0.f, 1.f, 0.f,
2678         0.f, 0.f, 0.f, 1.f,
2679     };
2680
2681     /* matrices for studio range */
2682     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion, in studio range */
2683     static const FLOAT COLORSPACE_BT601_TO_FULL[4*4] = {
2684         1.164383561643836f,                 0.f,  1.596026785714286f, 0.f,
2685         1.164383561643836f, -0.391762290094914f, -0.812967647237771f, 0.f,
2686         1.164383561643836f,  2.017232142857142f,                 0.f, 0.f,
2687                        0.f,                 0.f,                 0.f, 1.f,
2688     };
2689     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion, in studio range */
2690     static const FLOAT COLORSPACE_BT709_TO_FULL[4*4] = {
2691         1.164383561643836f,                 0.f,  1.792741071428571f, 0.f,
2692         1.164383561643836f, -0.213248614273730f, -0.532909328559444f, 0.f,
2693         1.164383561643836f,  2.112401785714286f,                 0.f, 0.f,
2694                        0.f,                 0.f,                 0.f, 1.f,
2695     };
2696     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.2020_conversion, in studio range */
2697     static const FLOAT COLORSPACE_BT2020_TO_FULL[4*4] = {
2698         1.164383561643836f,  0.000000000000f,  1.678674107143f, 0.f,
2699         1.164383561643836f, -0.127007098661f, -0.440987687946f, 0.f,
2700         1.164383561643836f,  2.141772321429f,  0.000000000000f, 0.f,
2701                        0.f,              0.f,              0.f, 1.f,
2702     };
2703
2704     PS_COLOR_TRANSFORM colorspace;
2705
2706     memcpy(colorspace.WhitePoint, IDENTITY_4X4, sizeof(colorspace.WhitePoint));
2707
2708     const FLOAT *ppColorspace;
2709     if (RGB_shader)
2710         ppColorspace = IDENTITY_4X4;
2711     else {
2712         switch (fmt->space){
2713             case COLOR_SPACE_BT709:
2714                 ppColorspace = COLORSPACE_BT709_TO_FULL;
2715                 break;
2716             case COLOR_SPACE_BT2020:
2717                 ppColorspace = COLORSPACE_BT2020_TO_FULL;
2718                 break;
2719             case COLOR_SPACE_BT601:
2720                 ppColorspace = COLORSPACE_BT601_TO_FULL;
2721                 break;
2722             default:
2723             case COLOR_SPACE_UNDEF:
2724                 if( fmt->i_height > 576 )
2725                     ppColorspace = COLORSPACE_BT709_TO_FULL;
2726                 else
2727                     ppColorspace = COLORSPACE_BT601_TO_FULL;
2728                 break;
2729         }
2730         /* all matrices work in studio range and output in full range */
2731         colorspace.WhitePoint[0*4 + 3] = -itu_black_level;
2732         colorspace.WhitePoint[1*4 + 3] = -itu_achromacy;
2733         colorspace.WhitePoint[2*4 + 3] = -itu_achromacy;
2734     }
2735
2736     memcpy(colorspace.Colorspace, ppColorspace, sizeof(colorspace.Colorspace));
2737
2738     constantInit.pSysMem = &colorspace;
2739
2740     static_assert((sizeof(PS_COLOR_TRANSFORM)%16)==0,"Constant buffers require 16-byte alignment");
2741     constantDesc.ByteWidth = sizeof(PS_COLOR_TRANSFORM);
2742     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, &constantInit, &quad->pPixelShaderConstants[1]);
2743     if(FAILED(hr)) {
2744         msg_Err(vd, "Could not create the pixel shader constant buffer. (hr=0x%lX)", hr);
2745         goto error;
2746     }
2747     quad->PSConstantsCount = 2;
2748
2749     /* vertex shader constant buffer */
2750     if ( projection == PROJECTION_MODE_EQUIRECTANGULAR )
2751     {
2752         constantDesc.ByteWidth = sizeof(VS_PROJECTION_CONST);
2753         static_assert((sizeof(VS_PROJECTION_CONST)%16)==0,"Constant buffers require 16-byte alignment");
2754         hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, NULL, &quad->pVertexShaderConstants);
2755         if(FAILED(hr)) {
2756             msg_Err(vd, "Could not create the vertex shader constant buffer. (hr=0x%lX)", hr);
2757             goto error;
2758         }
2759
2760         SetQuadVSProjection( vd, quad, &vd->cfg->viewpoint );
2761     }
2762
2763     quad->picSys.formatTexture = cfg->formatTexture;
2764     quad->picSys.context = sys->d3dcontext;
2765     ID3D11DeviceContext_AddRef(quad->picSys.context);
2766
2767     if (!AllocQuadVertices(vd, quad, projection))
2768         goto error;
2769     if (!UpdateQuadPosition(vd, quad, output, projection, orientation))
2770         goto error;
2771
2772     quad->d3dpixelShader = d3dpixelShader;
2773     if (projection == PROJECTION_MODE_RECTANGULAR)
2774         quad->d3dvertexShader = sys->flatVSShader;
2775     else
2776         quad->d3dvertexShader = sys->projectionVSShader;
2777
2778     return VLC_SUCCESS;
2779
2780 error:
2781     ReleaseQuad(quad);
2782     return VLC_EGENERIC;
2783 }
2784
2785 static void ReleaseQuad(d3d_quad_t *quad)
2786 {
2787     if (quad->pPixelShaderConstants[0])
2788     {
2789         ID3D11Buffer_Release(quad->pPixelShaderConstants[0]);
2790         quad->pPixelShaderConstants[0] = NULL;
2791     }
2792     if (quad->pPixelShaderConstants[1])
2793     {
2794         ID3D11Buffer_Release(quad->pPixelShaderConstants[1]);
2795         quad->pPixelShaderConstants[1] = NULL;
2796     }
2797     if (quad->pVertexBuffer)
2798     {
2799         ID3D11Buffer_Release(quad->pVertexBuffer);
2800         quad->pVertexBuffer = NULL;
2801     }
2802     quad->d3dvertexShader = NULL;
2803     if (quad->pIndexBuffer)
2804     {
2805         ID3D11Buffer_Release(quad->pIndexBuffer);
2806         quad->pIndexBuffer = NULL;
2807     }
2808     if (quad->pVertexShaderConstants)
2809     {
2810         ID3D11Buffer_Release(quad->pVertexShaderConstants);
2811         quad->pVertexShaderConstants = NULL;
2812     }
2813     ReleasePictureSys(&quad->picSys);
2814 }
2815
2816 static void Direct3D11DestroyResources(vout_display_t *vd)
2817 {
2818     vout_display_sys_t *sys = vd->sys;
2819
2820     Direct3D11DestroyPool(vd);
2821
2822     ReleaseQuad(&sys->picQuad);
2823     Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
2824     sys->d3dregion_count = 0;
2825
2826     ReleasePictureSys(&sys->stagingSys);
2827
2828     if (sys->flatVSShader)
2829     {
2830         ID3D11VertexShader_Release(sys->flatVSShader);
2831         sys->flatVSShader = NULL;
2832     }
2833     if (sys->projectionVSShader)
2834     {
2835         ID3D11VertexShader_Release(sys->projectionVSShader);
2836         sys->projectionVSShader = NULL;
2837     }
2838     if (sys->d3drenderTargetView)
2839     {
2840         ID3D11RenderTargetView_Release(sys->d3drenderTargetView);
2841         sys->d3drenderTargetView = NULL;
2842     }
2843     if (sys->d3ddepthStencilView)
2844     {
2845         ID3D11DepthStencilView_Release(sys->d3ddepthStencilView);
2846         sys->d3ddepthStencilView = NULL;
2847     }
2848     if (sys->pSPUPixelShader)
2849     {
2850         ID3D11PixelShader_Release(sys->pSPUPixelShader);
2851         sys->pSPUPixelShader = NULL;
2852     }
2853     if (sys->picQuadPixelShader)
2854     {
2855         ID3D11PixelShader_Release(sys->picQuadPixelShader);
2856         sys->picQuadPixelShader = NULL;
2857     }
2858 #if defined(HAVE_ID3D11VIDEODECODER)
2859     if( sys->context_lock != INVALID_HANDLE_VALUE )
2860     {
2861         CloseHandle( sys->context_lock );
2862         sys->context_lock = INVALID_HANDLE_VALUE;
2863     }
2864 #endif
2865
2866     msg_Dbg(vd, "Direct3D11 resources destroyed");
2867 }
2868
2869 static void Direct3D11DeleteRegions(int count, picture_t **region)
2870 {
2871     for (int i = 0; i < count; ++i) {
2872         if (region[i]) {
2873             picture_Release(region[i]);
2874         }
2875     }
2876     free(region);
2877 }
2878
2879 static void DestroyPictureQuad(picture_t *p_picture)
2880 {
2881     ReleaseQuad( (d3d_quad_t *) p_picture->p_sys );
2882     free( p_picture );
2883 }
2884
2885 static void UpdateQuadOpacity(vout_display_t *vd, const d3d_quad_t *quad, float opacity)
2886 {
2887     vout_display_sys_t *sys = vd->sys;
2888     D3D11_MAPPED_SUBRESOURCE mappedResource;
2889
2890     HRESULT hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pPixelShaderConstants[0], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2891     if (SUCCEEDED(hr)) {
2892         FLOAT *dst_data = mappedResource.pData;
2893         *dst_data = opacity;
2894         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pPixelShaderConstants[0], 0);
2895     }
2896     else {
2897         msg_Err(vd, "Failed to lock the subpicture vertex buffer (hr=0x%lX)", hr);
2898     }
2899 }
2900
2901 static int Direct3D11MapSubpicture(vout_display_t *vd, int *subpicture_region_count,
2902                                    picture_t ***region, subpicture_t *subpicture)
2903 {
2904     vout_display_sys_t *sys = vd->sys;
2905     D3D11_MAPPED_SUBRESOURCE mappedResource;
2906     D3D11_TEXTURE2D_DESC texDesc;
2907     HRESULT hr;
2908     int err;
2909
2910     if (sys->d3dregion_format == NULL)
2911         return VLC_EGENERIC;
2912
2913     int count = 0;
2914     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
2915         count++;
2916
2917     *region = calloc(count, sizeof(picture_t *));
2918     if (unlikely(*region==NULL))
2919         return VLC_ENOMEM;
2920     *subpicture_region_count = count;
2921
2922     int i = 0;
2923     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) {
2924         if (!r->fmt.i_width || !r->fmt.i_height)
2925             continue; // won't render anything, keep the cache for later
2926
2927         for (int j = 0; j < sys->d3dregion_count; j++) {
2928             picture_t *cache = sys->d3dregions[j];
2929             if (cache != NULL && ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX]) {
2930                 ID3D11Texture2D_GetDesc( ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX], &texDesc );
2931                 if (texDesc.Format == sys->d3dregion_format->formatTexture &&
2932                     texDesc.Width  == r->fmt.i_visible_width &&
2933                     texDesc.Height == r->fmt.i_visible_height) {
2934                     (*region)[i] = cache;
2935                     memset(&sys->d3dregions[j], 0, sizeof(cache)); // do not reuse this cached value
2936                     break;
2937                 }
2938             }
2939         }
2940
2941         picture_t *quad_picture = (*region)[i];
2942         if (quad_picture == NULL) {
2943             ID3D11Texture2D *textures[D3D11_MAX_SHADER_VIEW] = {0};
2944             d3d_quad_t *d3dquad = calloc(1, sizeof(*d3dquad));
2945             if (unlikely(d3dquad==NULL)) {
2946                 continue;
2947             }
2948             if (AllocateTextures(vd, sys->d3dregion_format, &r->fmt, 1, textures)) {
2949                 msg_Err(vd, "Failed to allocate %dx%d texture for OSD",
2950                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2951                 for (int j=0; j<D3D11_MAX_SHADER_VIEW; j++)
2952                     if (textures[j])
2953                         ID3D11Texture2D_Release(textures[j]);
2954                 free(d3dquad);
2955                 continue;
2956             }
2957
2958             for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++) {
2959                 d3dquad->picSys.texture[plane] = textures[plane];
2960             }
2961             if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->d3dregion_format,
2962                                    d3dquad->picSys.texture, 0,
2963                                    d3dquad->picSys.resourceView)) {
2964                 msg_Err(vd, "Failed to create %dx%d shader view for OSD",
2965                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2966                 free(d3dquad);
2967                 continue;
2968             }
2969             d3dquad->i_width    = r->fmt.i_width;
2970             d3dquad->i_height   = r->fmt.i_height;
2971             RECT output;
2972             output.left   = r->fmt.i_x_offset;
2973             output.right  = r->fmt.i_x_offset + r->fmt.i_width;
2974             output.top    = r->fmt.i_y_offset;
2975             output.bottom = r->fmt.i_y_offset + r->fmt.i_height;
2976
2977             err = SetupQuad( vd, &r->fmt, d3dquad, &output,
2978                              sys->d3dregion_format, sys->pSPUPixelShader,
2979                              PROJECTION_MODE_RECTANGULAR, ORIENT_NORMAL );
2980             if (err != VLC_SUCCESS) {
2981                 msg_Err(vd, "Failed to create %dx%d quad for OSD",
2982                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2983                 free(d3dquad);
2984                 continue;
2985             }
2986             picture_resource_t picres = {
2987                 .p_sys      = (picture_sys_t *) d3dquad,
2988                 .pf_destroy = DestroyPictureQuad,
2989             };
2990             (*region)[i] = picture_NewFromResource(&r->fmt, &picres);
2991             if ((*region)[i] == NULL) {
2992                 msg_Err(vd, "Failed to create %dx%d picture for OSD",
2993                         r->fmt.i_width, r->fmt.i_height);
2994                 ReleaseQuad(d3dquad);
2995                 continue;
2996             }
2997             quad_picture = (*region)[i];
2998         }
2999
3000         hr = ID3D11DeviceContext_Map(sys->d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
3001         if( SUCCEEDED(hr) ) {
3002             err = CommonUpdatePicture(quad_picture, NULL, mappedResource.pData, mappedResource.RowPitch);
3003             if (err != VLC_SUCCESS) {
3004                 msg_Err(vd, "Failed to set the buffer on the SPU picture" );
3005                 picture_Release(quad_picture);
3006                 continue;
3007             }
3008
3009             picture_CopyPixels(quad_picture, r->p_picture);
3010
3011             ID3D11DeviceContext_Unmap(sys->d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
3012         } else {
3013             msg_Err(vd, "Failed to map the SPU texture (hr=0x%lX)", hr );
3014             picture_Release(quad_picture);
3015             continue;
3016         }
3017
3018         d3d_quad_t *quad = (d3d_quad_t *) quad_picture->p_sys;
3019
3020         quad->cropViewport.Width =  (FLOAT) r->fmt.i_visible_width  * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
3021         quad->cropViewport.Height = (FLOAT) r->fmt.i_visible_height * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
3022         quad->cropViewport.MinDepth = 0.0f;
3023         quad->cropViewport.MaxDepth = 1.0f;
3024         quad->cropViewport.TopLeftX = sys->sys.rect_dest.left + (FLOAT) r->i_x * RECTWidth(sys->sys.rect_dest) / subpicture->i_original_picture_width;
3025         quad->cropViewport.TopLeftY = sys->sys.rect_dest.top  + (FLOAT) r->i_y * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
3026
3027         UpdateQuadOpacity(vd, quad, r->i_alpha / 255.0f );
3028     }
3029     return VLC_SUCCESS;
3030 }
3031