direct3d11: fix possibly failing CopySubresourceRegion on odd dimensions
[vlc.git] / modules / video_output / win32 / direct3d11.c
1 /*****************************************************************************
2  * direct3d11.c: Windows Direct3D11 video output module
3  *****************************************************************************
4  * Copyright (C) 2014-2015 VLC authors and VideoLAN
5  *
6  * Authors: Martell Malone <martellmalone@gmail.com>
7  *          Steve Lhomme <robux4@gmail.com>
8  *
9  * This program is free software; you can redistribute it and/or modify it
10  * under the terms of the GNU Lesser General Public License as published by
11  * the Free Software Foundation; either version 2.1 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  * GNU Lesser General Public License for more details.
18  *
19  * You should have received a copy of the GNU Lesser General Public License
20  * along with this program; if not, write to the Free Software Foundation,
21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22  *****************************************************************************/
23
24 #if !defined(_WIN32_WINNT) || _WIN32_WINNT < _WIN32_WINNT_WIN7
25 # undef _WIN32_WINNT
26 # define _WIN32_WINNT _WIN32_WINNT_WIN7
27 #endif
28
29 #ifdef HAVE_CONFIG_H
30 # include "config.h"
31 #endif
32
33 #include <vlc_common.h>
34 #include <vlc_plugin.h>
35 #include <vlc_vout_display.h>
36
37 #include <assert.h>
38 #include <math.h>
39
40 #define COBJMACROS
41 #include <initguid.h>
42 #include <d3d11.h>
43 #include <dxgi1_5.h>
44 #include <d3dcompiler.h>
45
46 /* avoided until we can pass ISwapchainPanel without c++/cx mode
47 # include <windows.ui.xaml.media.dxinterop.h> */
48
49 #include "../../video_chroma/d3d11_fmt.h"
50
51 #include "common.h"
52
53 #if !VLC_WINSTORE_APP
54 # define D3D11CreateDevice(args...)             sys->OurD3D11CreateDevice(args)
55 # define D3DCompile(args...)                    sys->OurD3DCompile(args)
56 #endif
57
58 DEFINE_GUID(GUID_SWAPCHAIN_WIDTH,  0xf1b59347, 0x1643, 0x411a, 0xad, 0x6b, 0xc7, 0x80, 0x17, 0x7a, 0x06, 0xb6);
59 DEFINE_GUID(GUID_SWAPCHAIN_HEIGHT, 0x6ea976a0, 0x9d60, 0x4bb7, 0xa5, 0xa9, 0x7d, 0xd1, 0x18, 0x7f, 0xc9, 0xbd);
60
61 static int  Open(vlc_object_t *);
62 static void Close(vlc_object_t *);
63
64 #define D3D11_HELP N_("Recommended video output for Windows 8 and later versions")
65 #define HW_BLENDING_TEXT N_("Use hardware blending support")
66 #define HW_BLENDING_LONGTEXT N_(\
67     "Try to use hardware acceleration for subtitle/OSD blending.")
68
69 vlc_module_begin ()
70     set_shortname("Direct3D11")
71     set_description(N_("Direct3D11 video output"))
72     set_help(D3D11_HELP)
73     set_category(CAT_VIDEO)
74     set_subcategory(SUBCAT_VIDEO_VOUT)
75
76     add_bool("direct3d11-hw-blending", true, HW_BLENDING_TEXT, HW_BLENDING_LONGTEXT, true)
77
78 #if VLC_WINSTORE_APP
79     add_integer("winrt-d3dcontext",    0x0, NULL, NULL, true); /* ID3D11DeviceContext* */
80     add_integer("winrt-swapchain",     0x0, NULL, NULL, true); /* IDXGISwapChain1*     */
81 #endif
82
83     set_capability("vout display", 240)
84     add_shortcut("direct3d11")
85     set_callbacks(Open, Close)
86 vlc_module_end ()
87
88 /* A Quad is texture that can be displayed in a rectangle */
89 typedef struct
90 {
91     picture_sys_t             picSys;
92     ID3D11Buffer              *pVertexBuffer;
93     UINT                      vertexCount;
94     ID3D11VertexShader        *d3dvertexShader;
95     ID3D11Buffer              *pIndexBuffer;
96     UINT                      indexCount;
97     ID3D11Buffer              *pVertexShaderConstants;
98     ID3D11Buffer              *pPixelShaderConstants[2];
99     UINT                       PSConstantsCount;
100     ID3D11PixelShader         *d3dpixelShader;
101     D3D11_VIEWPORT            cropViewport;
102     unsigned int              i_width;
103     unsigned int              i_height;
104 } d3d_quad_t;
105
106 typedef enum video_color_axis {
107     COLOR_AXIS_RGB,
108     COLOR_AXIS_YCBCR,
109 } video_color_axis;
110
111 typedef struct {
112     DXGI_COLOR_SPACE_TYPE   dxgi;
113     const char              *name;
114     video_color_axis        axis;
115     video_color_primaries_t primaries;
116     video_transfer_func_t   transfer;
117     video_color_space_t     color;
118     bool                    b_full_range;
119 } dxgi_color_space;
120
121 struct vout_display_sys_t
122 {
123     vout_display_sys_win32_t sys;
124
125     struct { /* TODO may go in vout_display_info_t without DXGI */
126         const dxgi_color_space   *colorspace;
127         unsigned                 luminance_peak;
128     } display;
129
130 #if !VLC_WINSTORE_APP
131     HINSTANCE                hdxgi_dll;        /* handle of the opened dxgi dll */
132     HINSTANCE                hd3d11_dll;       /* handle of the opened d3d11 dll */
133     HINSTANCE                hd3dcompiler_dll; /* handle of the opened d3dcompiler dll */
134     /* We should find a better way to store this or atleast a shorter name */
135     PFN_D3D11_CREATE_DEVICE_AND_SWAP_CHAIN OurD3D11CreateDeviceAndSwapChain;
136     PFN_D3D11_CREATE_DEVICE                OurD3D11CreateDevice;
137     pD3DCompile                            OurD3DCompile;
138 #endif
139 #if defined(HAVE_ID3D11VIDEODECODER)
140     HANDLE                   context_lock;     /* D3D11 Context lock necessary
141                                                   for hw decoding */
142 #endif
143     IDXGISwapChain1          *dxgiswapChain;   /* DXGI 1.1 swap chain */
144     IDXGISwapChain4          *dxgiswapChain4;  /* DXGI 1.5 for HDR */
145     ID3D11Device             *d3ddevice;       /* D3D device */
146     ID3D11DeviceContext      *d3dcontext;      /* D3D context */
147     d3d_quad_t               picQuad;
148     const d3d_format_t       *picQuadConfig;
149     ID3D11PixelShader        *picQuadPixelShader;
150
151     DXGI_FORMAT              decoderFormat;
152     picture_sys_t            stagingSys;
153
154     ID3D11RenderTargetView   *d3drenderTargetView;
155     ID3D11DepthStencilView   *d3ddepthStencilView;
156
157     ID3D11VertexShader        *flatVSShader;
158     ID3D11VertexShader        *projectionVSShader;
159
160     /* copy from the decoder pool into picSquad before display
161      * Uses a Texture2D with slices rather than a Texture2DArray for the decoder */
162     bool                     legacy_shader;
163
164     // SPU
165     vlc_fourcc_t             pSubpictureChromas[2];
166     ID3D11PixelShader        *pSPUPixelShader;
167     const d3d_format_t       *d3dregion_format;
168     int                      d3dregion_count;
169     picture_t                **d3dregions;
170 };
171
172 /* matches the D3D11_INPUT_ELEMENT_DESC we setup */
173 typedef struct d3d_vertex_t {
174     struct {
175         FLOAT x;
176         FLOAT y;
177         FLOAT z;
178     } position;
179     struct {
180         FLOAT x;
181         FLOAT y;
182     } texture;
183 } d3d_vertex_t;
184
185 typedef struct {
186     FLOAT Opacity;
187     FLOAT opacityPadding[3];
188 } PS_CONSTANT_BUFFER;
189
190 typedef struct {
191     FLOAT WhitePoint[4*4];
192     FLOAT Colorspace[4*4];
193 } PS_COLOR_TRANSFORM;
194
195 typedef struct {
196     FLOAT RotX[4*4];
197     FLOAT RotY[4*4];
198     FLOAT RotZ[4*4];
199     FLOAT View[4*4];
200     FLOAT Projection[4*4];
201 } VS_PROJECTION_CONST;
202
203 #define SPHERE_RADIUS 1.f
204
205 #define RECTWidth(r)   (int)((r).right - (r).left)
206 #define RECTHeight(r)  (int)((r).bottom - (r).top)
207
208 static picture_pool_t *Pool(vout_display_t *vd, unsigned count);
209
210 static void Prepare(vout_display_t *, picture_t *, subpicture_t *subpicture);
211 static void Display(vout_display_t *, picture_t *, subpicture_t *subpicture);
212
213 static HINSTANCE Direct3D11LoadShaderLibrary(void);
214 static void Direct3D11Destroy(vout_display_t *);
215
216 static int  Direct3D11Open (vout_display_t *, video_format_t *);
217 static void Direct3D11Close(vout_display_t *);
218
219 static int  Direct3D11CreateResources (vout_display_t *, video_format_t *);
220 static void Direct3D11DestroyResources(vout_display_t *);
221
222 static void Direct3D11DestroyPool(vout_display_t *);
223
224 static void DestroyDisplayPoolPicture(picture_t *);
225 static void Direct3D11DeleteRegions(int, picture_t **);
226 static int Direct3D11MapSubpicture(vout_display_t *, int *, picture_t ***, subpicture_t *);
227
228 static int SetupQuad(vout_display_t *, const video_format_t *, d3d_quad_t *, const RECT *,
229                      const d3d_format_t *, ID3D11PixelShader *, video_projection_mode_t,
230                      video_orientation_t);
231 static bool UpdateQuadPosition( vout_display_t *vd, d3d_quad_t *quad,
232                                 const RECT *output,
233                                 video_projection_mode_t projection,
234                                 video_orientation_t orientation );
235 static void ReleaseQuad(d3d_quad_t *);
236 static void UpdatePicQuadPosition(vout_display_t *);
237
238 static int Control(vout_display_t *vd, int query, va_list args);
239 static void Manage(vout_display_t *vd);
240
241 /* TODO: Move to a direct3d11_shaders header */
242 static const char* globVertexShaderFlat = "\
243   struct VS_INPUT\
244   {\
245     float4 Position   : POSITION;\
246     float4 Texture    : TEXCOORD0;\
247   };\
248   \
249   struct VS_OUTPUT\
250   {\
251     float4 Position   : SV_POSITION;\
252     float4 Texture    : TEXCOORD0;\
253   };\
254   \
255   VS_OUTPUT main( VS_INPUT In )\
256   {\
257     return In;\
258   }\
259 ";
260
261 #define STRINGIZE2(s) #s
262 #define STRINGIZE(s) STRINGIZE2(s)
263
264 static const char* globVertexShaderProjection = "\
265   cbuffer VS_PROJECTION_CONST : register(b0)\
266   {\
267      float4x4 RotX;\
268      float4x4 RotY;\
269      float4x4 RotZ;\
270      float4x4 View;\
271      float4x4 Projection;\
272   };\
273   struct VS_INPUT\
274   {\
275     float4 Position   : POSITION;\
276     float4 Texture    : TEXCOORD0;\
277   };\
278   \
279   struct VS_OUTPUT\
280   {\
281     float4 Position   : SV_POSITION;\
282     float4 Texture    : TEXCOORD0;\
283   };\
284   \
285   VS_OUTPUT main( VS_INPUT In )\
286   {\
287     VS_OUTPUT Output;\
288     float4 pos = In.Position;\
289     pos = mul(RotY, pos);\
290     pos = mul(RotX, pos);\
291     pos = mul(RotZ, pos);\
292     pos = mul(View, pos);\
293     pos = mul(Projection, pos);\
294     Output.Position = pos;\
295     Output.Texture = In.Texture;\
296     return Output;\
297   }\
298 ";
299
300 static const char* globPixelShaderDefault = "\
301   cbuffer PS_CONSTANT_BUFFER : register(b0)\
302   {\
303     float Opacity;\
304     float opacityPadding[3];\
305   };\
306   cbuffer PS_COLOR_TRANSFORM : register(b1)\
307   {\
308     float4x4 WhitePoint;\
309     float4x4 Colorspace;\
310   };\
311   Texture2D%s shaderTexture[" STRINGIZE(D3D11_MAX_SHADER_VIEW) "];\
312   SamplerState SampleType;\
313   \
314   struct PS_INPUT\
315   {\
316     float4 Position   : SV_POSITION;\
317     float4 Texture    : TEXCOORD0;\
318   };\
319   \
320   /* see http://filmicworlds.com/blog/filmic-tonemapping-operators/ */\
321   inline float hable(float x) {\
322       const float A = 0.15, B = 0.50, C = 0.10, D = 0.20, E = 0.02, F = 0.30;\
323       return ((x * (A*x + (C*B))+(D*E))/(x * (A*x + B) + (D*F))) - E/F;\
324   }\
325   \
326   inline float3 hable(float3 x) {\
327       x.r = hable(x.r);\
328       x.g = hable(x.g);\
329       x.b = hable(x.b);\
330       return x;\
331   }\
332   \
333   /* https://en.wikipedia.org/wiki/Hybrid_Log-Gamma#Technical_details */\
334   inline float inverse_HLG(float x){\
335       const float B67_a = 0.17883277;\
336       const float B67_b = 0.28466892;\
337       const float B67_c = 0.55991073;\
338       const float B67_inv_r2 = 4.0; /* 1/0.5² */\
339       if (x <= 0.5)\
340           x = x * x * B67_inv_r2;\
341       else\
342           x = exp((x - B67_c) / B67_a) + B67_b;\
343       return x;\
344   }\
345   \
346   inline float3 sourceToLinear(float3 rgb) {\
347       %s;\
348   }\
349   \
350   inline float3 linearToDisplay(float3 rgb) {\
351       %s;\
352   }\
353   \
354   inline float3 toneMapping(float3 rgb) {\
355       %s;\
356   }\
357   \
358   inline float3 adjustPeakLuminance(float3 rgb) {\
359       %s;\
360   }\
361   \
362   inline float3 adjustRange(float3 rgb) {\
363       %s;\
364   }\
365   \
366   float4 main( PS_INPUT In ) : SV_TARGET\
367   {\
368     float4 sample;\
369     \
370     %s /* sampling routine in sample */\
371     float4 rgba = mul(mul(sample, WhitePoint), Colorspace);\
372     float opacity = rgba.a * Opacity;\
373     float3 rgb = (float3)rgba;\
374     rgb = sourceToLinear(rgb);\
375     rgb = toneMapping(rgb);\
376     rgb = adjustPeakLuminance(rgb);\
377     rgb = linearToDisplay(rgb);\
378     rgb = adjustRange(rgb);\
379     return float4(rgb, saturate(opacity));\
380   }\
381 ";
382
383 #define ST2084_PQ_CONSTANTS  "const float ST2084_m1 = 2610.0 / (4096.0 * 4);\
384 const float ST2084_m2 = (2523.0 / 4096.0) * 128.0;\
385 const float ST2084_c1 = 3424.0 / 4096.0;\
386 const float ST2084_c2 = (2413.0 / 4096.0) * 32.0;\
387 const float ST2084_c3 = (2392.0 / 4096.0) * 32.0;"
388
389 #define DEFAULT_BRIGHTNESS 80
390
391
392 static int Direct3D11MapPoolTexture(picture_t *picture)
393 {
394     picture_sys_t *p_sys = picture->p_sys;
395     D3D11_MAPPED_SUBRESOURCE mappedResource;
396     HRESULT hr;
397     hr = ID3D11DeviceContext_Map(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
398     if( FAILED(hr) )
399     {
400         return VLC_EGENERIC;
401     }
402     return CommonUpdatePicture(picture, NULL, mappedResource.pData, mappedResource.RowPitch);
403 }
404
405 static void Direct3D11UnmapPoolTexture(picture_t *picture)
406 {
407     picture_sys_t *p_sys = picture->p_sys;
408     ID3D11DeviceContext_Unmap(p_sys->context, p_sys->resource[KNOWN_DXGI_INDEX], 0);
409 }
410
411 #if !VLC_WINSTORE_APP
412 static int OpenHwnd(vout_display_t *vd)
413 {
414     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
415     if (!sys)
416         return VLC_ENOMEM;
417
418     sys->hd3d11_dll = LoadLibrary(TEXT("D3D11.DLL"));
419     if (!sys->hd3d11_dll) {
420         msg_Warn(vd, "cannot load d3d11.dll, aborting");
421         return VLC_EGENERIC;
422     }
423
424     sys->hd3dcompiler_dll = Direct3D11LoadShaderLibrary();
425     if (!sys->hd3dcompiler_dll) {
426         msg_Err(vd, "cannot load d3dcompiler.dll, aborting");
427         Direct3D11Destroy(vd);
428         return VLC_EGENERIC;
429     }
430
431     sys->OurD3DCompile = (void *)GetProcAddress(sys->hd3dcompiler_dll, "D3DCompile");
432     if (!sys->OurD3DCompile) {
433         msg_Err(vd, "Cannot locate reference to D3DCompile in d3dcompiler DLL");
434         Direct3D11Destroy(vd);
435         return VLC_EGENERIC;
436     }
437
438     sys->OurD3D11CreateDevice =
439         (void *)GetProcAddress(sys->hd3d11_dll, "D3D11CreateDevice");
440     if (!sys->OurD3D11CreateDevice) {
441         msg_Err(vd, "Cannot locate reference to D3D11CreateDevice in d3d11 DLL");
442         Direct3D11Destroy(vd);
443         return VLC_EGENERIC;
444     }
445     return VLC_SUCCESS;
446 }
447 #else
448 static int OpenCoreW(vout_display_t *vd)
449 {
450     IDXGISwapChain1* dxgiswapChain  = var_InheritInteger(vd, "winrt-swapchain");
451     if (!dxgiswapChain)
452         return VLC_EGENERIC;
453     ID3D11DeviceContext* d3dcontext = var_InheritInteger(vd, "winrt-d3dcontext");
454     if (!d3dcontext)
455         return VLC_EGENERIC;
456     ID3D11Device* d3ddevice = NULL;
457     ID3D11DeviceContext_GetDevice(d3dcontext, &d3ddevice);
458     if (!d3ddevice)
459         return VLC_EGENERIC;
460
461     vout_display_sys_t *sys = vd->sys = calloc(1, sizeof(vout_display_sys_t));
462     if (!sys)
463         return VLC_ENOMEM;
464
465     sys->dxgiswapChain = dxgiswapChain;
466     sys->d3ddevice     = d3ddevice;
467     sys->d3dcontext    = d3dcontext;
468     IDXGISwapChain_AddRef     (sys->dxgiswapChain);
469     ID3D11Device_AddRef       (sys->d3ddevice);
470     ID3D11DeviceContext_AddRef(sys->d3dcontext);
471
472     return VLC_SUCCESS;
473 }
474 #endif
475
476 static bool is_d3d11_opaque(vlc_fourcc_t chroma)
477 {
478     switch (chroma)
479     {
480     case VLC_CODEC_D3D11_OPAQUE:
481     case VLC_CODEC_D3D11_OPAQUE_10B:
482         return true;
483     default:
484         return false;
485     }
486 }
487
488 #if VLC_WINSTORE_APP
489 static bool GetRect(const vout_display_sys_win32_t *p_sys, RECT *out)
490 {
491     const vout_display_sys_t *sys = (const vout_display_sys_t *)p_sys;
492     out->left   = 0;
493     out->top    = 0;
494     uint32_t i_width;
495     uint32_t i_height;
496     UINT dataSize = sizeof(i_width);
497     HRESULT hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_WIDTH, &dataSize, &i_width);
498     if (FAILED(hr)) {
499         return false;
500     }
501     dataSize = sizeof(i_height);
502     hr = IDXGISwapChain_GetPrivateData(sys->dxgiswapChain, &GUID_SWAPCHAIN_HEIGHT, &dataSize, &i_height);
503     if (FAILED(hr)) {
504         return false;
505     }
506     out->right  = i_width;
507     out->bottom = i_height;
508     return true;
509 }
510 #endif
511
512 static int Open(vlc_object_t *object)
513 {
514     vout_display_t *vd = (vout_display_t *)object;
515
516 #if !VLC_WINSTORE_APP
517     int ret = OpenHwnd(vd);
518 #else
519     int ret = OpenCoreW(vd);
520 #endif
521
522     if (ret != VLC_SUCCESS)
523         return ret;
524
525     if (CommonInit(vd))
526         goto error;
527
528 #if VLC_WINSTORE_APP
529     vd->sys->sys.pf_GetRect = GetRect;
530 #endif
531
532     video_format_t fmt;
533     video_format_Copy(&fmt, &vd->source);
534     if (Direct3D11Open(vd, &fmt)) {
535         msg_Err(vd, "Direct3D11 could not be opened");
536         goto error;
537     }
538
539     video_format_Clean(&vd->fmt);
540     vd->fmt = fmt;
541
542     vd->info.has_double_click     = true;
543     vd->info.has_hide_mouse       = false;
544     vd->info.has_pictures_invalid = vd->info.is_slow;
545
546     if (var_InheritBool(vd, "direct3d11-hw-blending") &&
547         vd->sys->d3dregion_format != NULL)
548     {
549         vd->sys->pSubpictureChromas[0] = vd->sys->d3dregion_format->fourcc;
550         vd->sys->pSubpictureChromas[1] = 0;
551         vd->info.subpicture_chromas = vd->sys->pSubpictureChromas;
552     }
553     else
554         vd->info.subpicture_chromas = NULL;
555
556     vd->pool    = Pool;
557     vd->prepare = Prepare;
558     vd->display = Display;
559     vd->control = Control;
560     vd->manage  = Manage;
561
562     msg_Dbg(vd, "Direct3D11 Open Succeeded");
563
564     return VLC_SUCCESS;
565
566 error:
567     Close(object);
568     return VLC_EGENERIC;
569 }
570
571 static void Close(vlc_object_t *object)
572 {
573     vout_display_t * vd = (vout_display_t *)object;
574
575     Direct3D11Close(vd);
576     CommonClean(vd);
577     Direct3D11Destroy(vd);
578     free(vd->sys);
579 }
580
581 static int AllocateTextures(vout_display_t *vd, const d3d_format_t *cfg,
582                             video_format_t *fmt, unsigned pool_size,
583                             ID3D11Texture2D *textures[],
584                             bool pool_type_display)
585 {
586     vout_display_sys_t *sys = vd->sys;
587     int plane;
588     HRESULT hr;
589     ID3D11Texture2D *slicedTexture = NULL;
590     D3D11_TEXTURE2D_DESC texDesc;
591     ZeroMemory(&texDesc, sizeof(texDesc));
592     texDesc.MipLevels = 1;
593     texDesc.SampleDesc.Count = 1;
594     texDesc.MiscFlags = 0; //D3D11_RESOURCE_MISC_SHARED;
595     texDesc.Format = cfg->formatTexture;
596     texDesc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
597     if (is_d3d11_opaque(fmt->i_chroma)) {
598         texDesc.BindFlags |= D3D11_BIND_DECODER;
599         texDesc.Usage = D3D11_USAGE_DEFAULT;
600         texDesc.CPUAccessFlags = 0;
601     } else {
602         texDesc.Usage = D3D11_USAGE_DYNAMIC;
603         texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
604     }
605     texDesc.ArraySize = pool_size;
606     texDesc.Height = fmt->i_height;
607     texDesc.Width = fmt->i_width;
608
609     hr = ID3D11Device_CreateTexture2D( sys->d3ddevice, &texDesc, NULL, &slicedTexture );
610     if (FAILED(hr)) {
611         msg_Err(vd, "CreateTexture2D failed for the %d pool. (hr=0x%0lx)", pool_size, hr);
612         goto error;
613     }
614
615     for (unsigned picture_count = 0; picture_count < pool_size; picture_count++) {
616         textures[picture_count * D3D11_MAX_SHADER_VIEW] = slicedTexture;
617         ID3D11Texture2D_AddRef(slicedTexture);
618
619         for (plane = 1; plane < D3D11_MAX_SHADER_VIEW; plane++) {
620             if (!cfg->resourceFormat[plane])
621                 textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] = NULL;
622             else
623             {
624                 textures[picture_count * D3D11_MAX_SHADER_VIEW + plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW];
625                 ID3D11Texture2D_AddRef(textures[picture_count * D3D11_MAX_SHADER_VIEW + plane]);
626             }
627         }
628     }
629
630     if (!is_d3d11_opaque(fmt->i_chroma)) {
631         D3D11_MAPPED_SUBRESOURCE mappedResource;
632         const vlc_chroma_description_t *p_chroma_desc = vlc_fourcc_GetChromaDescription( fmt->i_chroma );
633
634         hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource*)textures[0], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
635         if( FAILED(hr) ) {
636             msg_Err(vd, "The texture cannot be mapped. (hr=0x%lX)", hr);
637             goto error;
638         }
639         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource*)textures[0], 0);
640         if (mappedResource.RowPitch < p_chroma_desc->pixel_size * texDesc.Width) {
641             msg_Err( vd, "The texture row pitch is too small (%d instead of %d)", mappedResource.RowPitch,
642                      p_chroma_desc->pixel_size * texDesc.Width );
643             goto error;
644         }
645     }
646
647     if (slicedTexture)
648         ID3D11Texture2D_Release(slicedTexture);
649     return VLC_SUCCESS;
650 error:
651     if (slicedTexture)
652         ID3D11Texture2D_Release(slicedTexture);
653     return VLC_EGENERIC;
654 }
655
656 static picture_pool_t *Pool(vout_display_t *vd, unsigned pool_size)
657 {
658     /* compensate for extra hardware decoding pulling extra pictures from our pool */
659     pool_size += 2;
660
661     vout_display_sys_t *sys = vd->sys;
662     ID3D11Texture2D  *textures[pool_size * D3D11_MAX_SHADER_VIEW];
663     picture_t **pictures = NULL;
664     picture_t *picture;
665     unsigned  plane;
666     unsigned  picture_count = 0;
667     picture_pool_configuration_t pool_cfg = {};
668
669     if (sys->sys.pool)
670         return sys->sys.pool;
671
672     if (vd->info.is_slow)
673         pool_size = 1;
674
675     video_format_t surface_fmt = vd->fmt;
676     if (is_d3d11_opaque(surface_fmt.i_chroma))
677     {
678         /* worst case scenario we need 128 alignment for HEVC */
679         surface_fmt.i_width  = (surface_fmt.i_width  + 0x7F) & ~0x7F;
680         surface_fmt.i_height = (surface_fmt.i_height + 0x7F) & ~0x7F;
681     }
682     else if ( sys->picQuadConfig->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
683               sys->picQuadConfig->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
684     {
685         surface_fmt.i_width  = (surface_fmt.i_width  + 0x01) & ~0x01;
686         surface_fmt.i_height = (surface_fmt.i_height + 0x01) & ~0x01;
687     }
688
689     if (AllocateTextures(vd, sys->picQuadConfig, &surface_fmt, pool_size, textures,
690                          true))
691         goto error;
692
693     if (!vd->info.is_slow) {
694         HRESULT           hr;
695         ID3D10Multithread *pMultithread;
696         hr = ID3D11Device_QueryInterface( sys->d3ddevice, &IID_ID3D10Multithread, (void **)&pMultithread);
697         if (SUCCEEDED(hr)) {
698             ID3D10Multithread_SetMultithreadProtected(pMultithread, TRUE);
699             ID3D10Multithread_Release(pMultithread);
700         }
701     }
702
703     pictures = calloc(pool_size, sizeof(*pictures));
704     if (!pictures)
705         goto error;
706
707     for (picture_count = 0; picture_count < pool_size; picture_count++) {
708         picture_sys_t *picsys = calloc(1, sizeof(*picsys));
709         if (unlikely(picsys == NULL))
710             goto error;
711
712         for (plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
713             picsys->texture[plane] = textures[picture_count * D3D11_MAX_SHADER_VIEW + plane];
714
715         picsys->slice_index = picture_count;
716         picsys->formatTexture = sys->picQuadConfig->formatTexture;
717         picsys->decoderFormat = sys->decoderFormat;
718         picsys->context = sys->d3dcontext;
719
720         picture_resource_t resource = {
721             .p_sys = picsys,
722             .pf_destroy = DestroyDisplayPoolPicture,
723         };
724
725         picture = picture_NewFromResource(&surface_fmt, &resource);
726         if (unlikely(picture == NULL)) {
727             free(picsys);
728             msg_Err( vd, "Failed to create picture %d in the pool.", picture_count );
729             goto error;
730         }
731
732         pictures[picture_count] = picture;
733         /* each picture_t holds a ref to the context and release it on Destroy */
734         ID3D11DeviceContext_AddRef(picsys->context);
735     }
736
737 #ifdef HAVE_ID3D11VIDEODECODER
738     if (!is_d3d11_opaque(surface_fmt.i_chroma) || sys->legacy_shader)
739     {
740         /* we need a staging texture */
741         video_format_t staging_fmt = surface_fmt;
742         if ( sys->picQuadConfig->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
743              sys->picQuadConfig->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
744         {
745             staging_fmt.i_width  = (staging_fmt.i_width  + 0x01) & ~0x01;
746             staging_fmt.i_height = (staging_fmt.i_height + 0x01) & ~0x01;
747         }
748         if (AllocateTextures(vd, sys->picQuadConfig, &staging_fmt, 1, textures, true))
749             goto error;
750
751         sys->picQuad.i_width    = staging_fmt.i_width;
752         sys->picQuad.i_height   = staging_fmt.i_height;
753
754         for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++)
755             sys->stagingSys.texture[plane] = textures[plane];
756
757         if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->picQuadConfig,
758                                textures, 0, sys->stagingSys.resourceView))
759             goto error;
760     } else
761 #endif
762     {
763         sys->picQuad.i_width    = surface_fmt.i_width;
764         sys->picQuad.i_height   = surface_fmt.i_height;
765
766         for (picture_count = 0; picture_count < pool_size; picture_count++) {
767             if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->picQuadConfig,
768                                    pictures[picture_count]->p_sys->texture, picture_count,
769                                    pictures[picture_count]->p_sys->resourceView))
770                 goto error;
771         }
772     }
773
774     if (SetupQuad( vd, &surface_fmt, &sys->picQuad, &sys->sys.rect_src_clipped,
775                    sys->picQuadConfig, sys->picQuadPixelShader,
776                    surface_fmt.projection_mode, vd->fmt.orientation ) != VLC_SUCCESS) {
777         msg_Err(vd, "Could not Create the main quad picture.");
778         return NULL;
779     }
780
781     if (vd->info.is_slow) {
782         pool_cfg.lock          = Direct3D11MapPoolTexture;
783         //pool_cfg.unlock        = Direct3D11UnmapPoolTexture;
784     }
785     pool_cfg.picture       = pictures;
786     pool_cfg.picture_count = pool_size;
787     sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
788
789 error:
790     if (sys->sys.pool == NULL) {
791         if (pictures) {
792             msg_Dbg(vd, "Failed to create the picture d3d11 pool");
793             for (unsigned i=0;i<picture_count; ++i)
794                 picture_Release(pictures[i]);
795             free(pictures);
796         }
797
798         /* create an empty pool to avoid crashing */
799         pool_cfg.picture_count = 0;
800         sys->sys.pool = picture_pool_NewExtended( &pool_cfg );
801     } else {
802         msg_Dbg(vd, "D3D11 pool succeed with %d surfaces (%dx%d) context 0x%p",
803                 picture_count, surface_fmt.i_width, surface_fmt.i_height, sys->d3dcontext);
804     }
805     return sys->sys.pool;
806 }
807
808 static void DestroyDisplayPoolPicture(picture_t *picture)
809 {
810     picture_sys_t *p_sys = picture->p_sys;
811     ReleasePictureSys( p_sys );
812     free(p_sys);
813     free(picture);
814 }
815
816 static HRESULT UpdateBackBuffer(vout_display_t *vd)
817 {
818     vout_display_sys_t *sys = vd->sys;
819     HRESULT hr;
820     ID3D11Texture2D* pDepthStencil;
821     ID3D11Texture2D* pBackBuffer;
822     RECT rect;
823 #if VLC_WINSTORE_APP
824     if (!GetRect(&sys->sys, &rect))
825 #endif
826         rect = sys->sys.rect_dest_clipped;
827     uint32_t i_width = RECTWidth(rect);
828     uint32_t i_height = RECTHeight(rect);
829
830     if (sys->d3drenderTargetView) {
831         ID3D11RenderTargetView_Release(sys->d3drenderTargetView);
832         sys->d3drenderTargetView = NULL;
833     }
834     if (sys->d3ddepthStencilView) {
835         ID3D11DepthStencilView_Release(sys->d3ddepthStencilView);
836         sys->d3ddepthStencilView = NULL;
837     }
838
839     /* TODO detect is the size is the same as the output and switch to fullscreen mode */
840     hr = IDXGISwapChain_ResizeBuffers(sys->dxgiswapChain, 0, i_width, i_height,
841         DXGI_FORMAT_UNKNOWN, 0);
842     if (FAILED(hr)) {
843        msg_Err(vd, "Failed to resize the backbuffer. (hr=0x%lX)", hr);
844        return hr;
845     }
846
847     hr = IDXGISwapChain_GetBuffer(sys->dxgiswapChain, 0, &IID_ID3D11Texture2D, (LPVOID *)&pBackBuffer);
848     if (FAILED(hr)) {
849        msg_Err(vd, "Could not get the backbuffer for the Swapchain. (hr=0x%lX)", hr);
850        return hr;
851     }
852
853     hr = ID3D11Device_CreateRenderTargetView(sys->d3ddevice, (ID3D11Resource *)pBackBuffer, NULL, &sys->d3drenderTargetView);
854     ID3D11Texture2D_Release(pBackBuffer);
855     if (FAILED(hr)) {
856         msg_Err(vd, "Failed to create the target view. (hr=0x%lX)", hr);
857         return hr;
858     }
859
860     D3D11_TEXTURE2D_DESC deptTexDesc;
861     memset(&deptTexDesc, 0,sizeof(deptTexDesc));
862     deptTexDesc.ArraySize = 1;
863     deptTexDesc.BindFlags = D3D11_BIND_DEPTH_STENCIL;
864     deptTexDesc.CPUAccessFlags = 0;
865     deptTexDesc.Format = DXGI_FORMAT_D24_UNORM_S8_UINT;
866     deptTexDesc.Width = i_width;
867     deptTexDesc.Height = i_height;
868     deptTexDesc.MipLevels = 1;
869     deptTexDesc.MiscFlags = 0;
870     deptTexDesc.SampleDesc.Count = 1;
871     deptTexDesc.SampleDesc.Quality = 0;
872     deptTexDesc.Usage = D3D11_USAGE_DEFAULT;
873
874     hr = ID3D11Device_CreateTexture2D(sys->d3ddevice, &deptTexDesc, NULL, &pDepthStencil);
875     if (FAILED(hr)) {
876        msg_Err(vd, "Could not create the depth stencil texture. (hr=0x%lX)", hr);
877        return hr;
878     }
879
880     D3D11_DEPTH_STENCIL_VIEW_DESC depthViewDesc;
881     memset(&depthViewDesc, 0, sizeof(depthViewDesc));
882
883     depthViewDesc.Format = deptTexDesc.Format;
884     depthViewDesc.ViewDimension = D3D11_DSV_DIMENSION_TEXTURE2D;
885     depthViewDesc.Texture2D.MipSlice = 0;
886
887     hr = ID3D11Device_CreateDepthStencilView(sys->d3ddevice, (ID3D11Resource *)pDepthStencil, &depthViewDesc, &sys->d3ddepthStencilView);
888     ID3D11Texture2D_Release(pDepthStencil);
889
890     if (FAILED(hr)) {
891        msg_Err(vd, "Could not create the depth stencil view. (hr=0x%lX)", hr);
892        return hr;
893     }
894
895     return S_OK;
896 }
897
898 /* rotation around the Z axis */
899 static void getZRotMatrix(float theta, FLOAT matrix[static 16])
900 {
901     float st, ct;
902
903     sincosf(theta, &st, &ct);
904
905     const FLOAT m[] = {
906     /*  x    y    z    w */
907         ct,  -st, 0.f, 0.f,
908         st,  ct,  0.f, 0.f,
909         0.f, 0.f, 1.f, 0.f,
910         0.f, 0.f, 0.f, 1.f
911     };
912
913     memcpy(matrix, m, sizeof(m));
914 }
915
916 /* rotation around the Y axis */
917 static void getYRotMatrix(float theta, FLOAT matrix[static 16])
918 {
919     float st, ct;
920
921     sincosf(theta, &st, &ct);
922
923     const FLOAT m[] = {
924     /*  x    y    z    w */
925         ct,  0.f, -st, 0.f,
926         0.f, 1.f, 0.f, 0.f,
927         st,  0.f, ct,  0.f,
928         0.f, 0.f, 0.f, 1.f
929     };
930
931     memcpy(matrix, m, sizeof(m));
932 }
933
934 /* rotation around the X axis */
935 static void getXRotMatrix(float phi, FLOAT matrix[static 16])
936 {
937     float sp, cp;
938
939     sincosf(phi, &sp, &cp);
940
941     const FLOAT m[] = {
942     /*  x    y    z    w */
943         1.f, 0.f, 0.f, 0.f,
944         0.f, cp,  sp,  0.f,
945         0.f, -sp, cp,  0.f,
946         0.f, 0.f, 0.f, 1.f
947     };
948
949     memcpy(matrix, m, sizeof(m));
950 }
951
952 static void getZoomMatrix(float zoom, FLOAT matrix[static 16]) {
953
954     const FLOAT m[] = {
955         /* x   y     z     w */
956         1.0f, 0.0f, 0.0f, 0.0f,
957         0.0f, 1.0f, 0.0f, 0.0f,
958         0.0f, 0.0f, 1.0f, 0.0f,
959         0.0f, 0.0f, zoom, 1.0f
960     };
961
962     memcpy(matrix, m, sizeof(m));
963 }
964
965 /* perspective matrix see https://www.opengl.org/sdk/docs/man2/xhtml/gluPerspective.xml */
966 static void getProjectionMatrix(float sar, float fovy, FLOAT matrix[static 16]) {
967
968     float zFar  = 1000;
969     float zNear = 0.01;
970
971     float f = 1.f / tanf(fovy / 2.f);
972
973     const FLOAT m[] = {
974         f / sar, 0.f,                   0.f,                0.f,
975         0.f,     f,                     0.f,                0.f,
976         0.f,     0.f,     (zNear + zFar) / (zNear - zFar), -1.f,
977         0.f,     0.f, (2 * zNear * zFar) / (zNear - zFar),  0.f};
978
979      memcpy(matrix, m, sizeof(m));
980 }
981
982 static float UpdateFOVy(float f_fovx, float f_sar)
983 {
984     return 2 * atanf(tanf(f_fovx / 2) / f_sar);
985 }
986
987 static float UpdateZ(float f_fovx, float f_fovy)
988 {
989     /* Do trigonometry to calculate the minimal z value
990      * that will allow us to zoom out without seeing the outside of the
991      * sphere (black borders). */
992     float tan_fovx_2 = tanf(f_fovx / 2);
993     float tan_fovy_2 = tanf(f_fovy / 2);
994     float z_min = - SPHERE_RADIUS / sinf(atanf(sqrtf(
995                     tan_fovx_2 * tan_fovx_2 + tan_fovy_2 * tan_fovy_2)));
996
997     /* The FOV value above which z is dynamically calculated. */
998     const float z_thresh = 90.f;
999
1000     float f_z;
1001     if (f_fovx <= z_thresh * M_PI / 180)
1002         f_z = 0;
1003     else
1004     {
1005         float f = z_min / ((FIELD_OF_VIEW_DEGREES_MAX - z_thresh) * M_PI / 180);
1006         f_z = f * f_fovx - f * z_thresh * M_PI / 180;
1007         if (f_z < z_min)
1008             f_z = z_min;
1009     }
1010     return f_z;
1011 }
1012
1013 static void SetQuadVSProjection(vout_display_t *vd, d3d_quad_t *quad, const vlc_viewpoint_t *p_vp)
1014 {
1015     if (!quad->pVertexShaderConstants)
1016         return;
1017
1018 #define RAD(d) ((float) ((d) * M_PI / 180.f))
1019     float f_fovx = RAD(p_vp->fov);
1020     if ( f_fovx > FIELD_OF_VIEW_DEGREES_MAX * M_PI / 180 + 0.001f ||
1021          f_fovx < -0.001f )
1022         return;
1023
1024     float f_sar = (float) vd->cfg->display.width / vd->cfg->display.height;
1025     float f_teta = RAD(p_vp->yaw) - (float) M_PI_2;
1026     float f_phi  = RAD(p_vp->pitch);
1027     float f_roll = RAD(p_vp->roll);
1028     float f_fovy = UpdateFOVy(f_fovx, f_sar);
1029     float f_z = UpdateZ(f_fovx, f_fovy);
1030
1031     vout_display_sys_t *sys = vd->sys;
1032     HRESULT hr;
1033     D3D11_MAPPED_SUBRESOURCE mapped;
1034     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped);
1035     if (SUCCEEDED(hr)) {
1036         VS_PROJECTION_CONST *dst_data = mapped.pData;
1037         getXRotMatrix(f_phi, dst_data->RotX);
1038         getYRotMatrix(f_teta,   dst_data->RotY);
1039         getZRotMatrix(f_roll,  dst_data->RotZ);
1040         getZoomMatrix(SPHERE_RADIUS * f_z, dst_data->View);
1041         getProjectionMatrix(f_sar, f_fovy, dst_data->Projection);
1042     }
1043     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexShaderConstants, 0);
1044 #undef RAD
1045 }
1046
1047 static void UpdateSize(vout_display_t *vd)
1048 {
1049     vout_display_sys_t *sys = vd->sys;
1050 #if defined(HAVE_ID3D11VIDEODECODER)
1051     if( sys->context_lock != INVALID_HANDLE_VALUE )
1052     {
1053         WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1054     }
1055 #endif
1056     msg_Dbg(vd, "Detected size change %dx%d", RECTWidth(sys->sys.rect_dest_clipped),
1057             RECTHeight(sys->sys.rect_dest_clipped));
1058
1059     UpdateBackBuffer(vd);
1060
1061     UpdatePicQuadPosition(vd);
1062
1063     sys->picQuad.i_width = RECTWidth(sys->sys.rect_src);
1064     sys->picQuad.i_height = RECTHeight(sys->sys.rect_src);
1065     UpdateQuadPosition(vd, &sys->picQuad, &sys->sys.rect_src_clipped,
1066                        vd->fmt.projection_mode, vd->fmt.orientation);
1067
1068 #if defined(HAVE_ID3D11VIDEODECODER)
1069     if( sys->context_lock != INVALID_HANDLE_VALUE )
1070     {
1071         ReleaseMutex( sys->context_lock );
1072     }
1073 #endif
1074 }
1075
1076 static inline bool RectEquals(const RECT *r1, const RECT *r2)
1077 {
1078     return r1->bottom == r2->bottom && r1->top == r2->top &&
1079            r1->left == r2->left && r1->right == r2->right;
1080 }
1081
1082 static int Control(vout_display_t *vd, int query, va_list args)
1083 {
1084     vout_display_sys_t *sys = vd->sys;
1085     RECT before_src_clipped  = sys->sys.rect_src_clipped;
1086     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
1087     RECT before_dest         = sys->sys.rect_dest;
1088
1089     int res = CommonControl( vd, query, args );
1090
1091     if (query == VOUT_DISPLAY_CHANGE_VIEWPOINT)
1092     {
1093         const vout_display_cfg_t *cfg = va_arg(args, const vout_display_cfg_t*);
1094         if ( sys->picQuad.pVertexShaderConstants )
1095         {
1096             SetQuadVSProjection( vd, &sys->picQuad, &cfg->viewpoint );
1097             res = VLC_SUCCESS;
1098         }
1099     }
1100
1101     if (!RectEquals(&before_src_clipped,  &sys->sys.rect_src_clipped) ||
1102         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
1103         !RectEquals(&before_dest,         &sys->sys.rect_dest) )
1104     {
1105         UpdateSize(vd);
1106     }
1107
1108     return res;
1109 }
1110
1111 static void Manage(vout_display_t *vd)
1112 {
1113     vout_display_sys_t *sys = vd->sys;
1114     RECT before_src_clipped  = sys->sys.rect_src_clipped;
1115     RECT before_dest_clipped = sys->sys.rect_dest_clipped;
1116     RECT before_dest         = sys->sys.rect_dest;
1117
1118     CommonManage(vd);
1119
1120     if (!RectEquals(&before_src_clipped, &sys->sys.rect_src_clipped) ||
1121         !RectEquals(&before_dest_clipped, &sys->sys.rect_dest_clipped) ||
1122         !RectEquals(&before_dest, &sys->sys.rect_dest))
1123     {
1124         UpdateSize(vd);
1125     }
1126 }
1127
1128 static void Prepare(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
1129 {
1130     vout_display_sys_t *sys = vd->sys;
1131
1132 #if defined(HAVE_ID3D11VIDEODECODER)
1133     if( sys->context_lock != INVALID_HANDLE_VALUE )
1134         WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1135 #endif
1136     picture_sys_t *p_sys = ActivePictureSys(picture);
1137     if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader) {
1138         D3D11_TEXTURE2D_DESC texDesc;
1139         if (!is_d3d11_opaque(picture->format.i_chroma))
1140             Direct3D11UnmapPoolTexture(picture);
1141         ID3D11Texture2D_GetDesc(sys->stagingSys.texture[0], &texDesc);
1142         D3D11_BOX box = {
1143             .top = 0,
1144             .bottom = picture->format.i_y_offset + picture->format.i_visible_height,
1145             .left = 0,
1146             .right = picture->format.i_x_offset + picture->format.i_visible_width,
1147             .back = 1,
1148         };
1149         if ( sys->picQuadConfig->formatTexture != DXGI_FORMAT_R8G8B8A8_UNORM &&
1150              sys->picQuadConfig->formatTexture != DXGI_FORMAT_B5G6R5_UNORM )
1151         {
1152             box.bottom = (box.bottom + 0x01) & ~0x01;
1153             box.right  = (box.right  + 0x01) & ~0x01;
1154         }
1155         assert(box.right <= texDesc.Width);
1156         assert(box.bottom <= texDesc.Height);
1157         ID3D11DeviceContext_CopySubresourceRegion(sys->d3dcontext,
1158                                                   sys->stagingSys.resource[KNOWN_DXGI_INDEX],
1159                                                   0, 0, 0, 0,
1160                                                   p_sys->resource[KNOWN_DXGI_INDEX],
1161                                                   p_sys->slice_index, &box);
1162     }
1163     else
1164     {
1165         D3D11_TEXTURE2D_DESC texDesc;
1166         ID3D11Texture2D_GetDesc(p_sys->texture[0], &texDesc);
1167         if (texDesc.BindFlags & D3D11_BIND_SHADER_RESOURCE)
1168         {
1169             /* for performance reason we don't want to allocate this during
1170              * display, do it preferrably when creating the texture */
1171             assert(p_sys->resourceView[0]!=NULL);
1172         }
1173
1174         if ( vd->fmt.i_height != texDesc.Height ||
1175              vd->fmt.i_width != texDesc.Width )
1176         {
1177             /* the decoder produced different sizes than the vout, we need to
1178              * adjust the vertex */
1179             vd->fmt.i_height = texDesc.Height;
1180             vd->fmt.i_width = texDesc.Width;
1181             UpdateRects(vd, NULL, NULL, true);
1182             UpdateSize(vd);
1183         }
1184     }
1185
1186     if (subpicture) {
1187         int subpicture_region_count    = 0;
1188         picture_t **subpicture_regions = NULL;
1189         Direct3D11MapSubpicture(vd, &subpicture_region_count, &subpicture_regions, subpicture);
1190         Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
1191         sys->d3dregion_count = subpicture_region_count;
1192         sys->d3dregions      = subpicture_regions;
1193     }
1194
1195     ID3D11DeviceContext_Flush(sys->d3dcontext);
1196 #if defined(HAVE_ID3D11VIDEODECODER)
1197     if ( sys->context_lock != INVALID_HANDLE_VALUE)
1198         ReleaseMutex( sys->context_lock );
1199 #endif
1200 }
1201
1202 static void DisplayD3DPicture(vout_display_sys_t *sys, d3d_quad_t *quad, ID3D11ShaderResourceView *resourceView[D3D11_MAX_SHADER_VIEW])
1203 {
1204     UINT stride = sizeof(d3d_vertex_t);
1205     UINT offset = 0;
1206
1207     /* Render the quad */
1208     /* vertex shader */
1209     ID3D11DeviceContext_IASetVertexBuffers(sys->d3dcontext, 0, 1, &quad->pVertexBuffer, &stride, &offset);
1210     ID3D11DeviceContext_IASetIndexBuffer(sys->d3dcontext, quad->pIndexBuffer, DXGI_FORMAT_R16_UINT, 0);
1211     if ( quad->pVertexShaderConstants )
1212         ID3D11DeviceContext_VSSetConstantBuffers(sys->d3dcontext, 0, 1, &quad->pVertexShaderConstants);
1213
1214     ID3D11DeviceContext_VSSetShader(sys->d3dcontext, quad->d3dvertexShader, NULL, 0);
1215
1216     /* pixel shader */
1217     ID3D11DeviceContext_PSSetShader(sys->d3dcontext, quad->d3dpixelShader, NULL, 0);
1218
1219     ID3D11DeviceContext_PSSetConstantBuffers(sys->d3dcontext, 0, quad->PSConstantsCount, quad->pPixelShaderConstants);
1220     ID3D11DeviceContext_PSSetShaderResources(sys->d3dcontext, 0, D3D11_MAX_SHADER_VIEW, resourceView);
1221
1222     ID3D11DeviceContext_RSSetViewports(sys->d3dcontext, 1, &quad->cropViewport);
1223
1224     ID3D11DeviceContext_DrawIndexed(sys->d3dcontext, quad->indexCount, 0, 0);
1225 }
1226
1227 static void Display(vout_display_t *vd, picture_t *picture, subpicture_t *subpicture)
1228 {
1229     vout_display_sys_t *sys = vd->sys;
1230 #ifdef HAVE_ID3D11VIDEODECODER
1231     if (sys->context_lock != INVALID_HANDLE_VALUE && is_d3d11_opaque(picture->format.i_chroma))
1232     {
1233         WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1234     }
1235 #endif
1236
1237     FLOAT blackRGBA[4] = {0.0f, 0.0f, 0.0f, 1.0f};
1238     ID3D11DeviceContext_ClearRenderTargetView(sys->d3dcontext, sys->d3drenderTargetView, blackRGBA);
1239
1240     /* no ID3D11Device operations should come here */
1241
1242     ID3D11DeviceContext_OMSetRenderTargets(sys->d3dcontext, 1, &sys->d3drenderTargetView, sys->d3ddepthStencilView);
1243
1244     ID3D11DeviceContext_ClearDepthStencilView(sys->d3dcontext, sys->d3ddepthStencilView, D3D11_CLEAR_DEPTH | D3D11_CLEAR_STENCIL, 1.0f, 0);
1245
1246     /* Render the quad */
1247     if (!is_d3d11_opaque(picture->format.i_chroma) || sys->legacy_shader)
1248         DisplayD3DPicture(sys, &sys->picQuad, sys->stagingSys.resourceView);
1249     else {
1250         picture_sys_t *p_sys = ActivePictureSys(picture);
1251         DisplayD3DPicture(sys, &sys->picQuad, p_sys->resourceView);
1252     }
1253
1254     if (subpicture) {
1255         // draw the additional vertices
1256         for (int i = 0; i < sys->d3dregion_count; ++i) {
1257             if (sys->d3dregions[i])
1258             {
1259                 d3d_quad_t *quad = (d3d_quad_t *) sys->d3dregions[i]->p_sys;
1260                 DisplayD3DPicture(sys, quad, quad->picSys.resourceView);
1261             }
1262         }
1263     }
1264
1265 #if defined(HAVE_ID3D11VIDEODECODER)
1266     if (sys->context_lock != INVALID_HANDLE_VALUE && is_d3d11_opaque(picture->format.i_chroma))
1267     {
1268         ReleaseMutex( sys->context_lock );
1269     }
1270 #endif
1271
1272     if (sys->dxgiswapChain4 && picture->format.mastering.max_luminance)
1273     {
1274         DXGI_HDR_METADATA_HDR10 hdr10 = {0};
1275         hdr10.GreenPrimary[0] = picture->format.mastering.primaries[0];
1276         hdr10.GreenPrimary[1] = picture->format.mastering.primaries[1];
1277         hdr10.BluePrimary[0]  = picture->format.mastering.primaries[2];
1278         hdr10.BluePrimary[1]  = picture->format.mastering.primaries[3];
1279         hdr10.RedPrimary[0]   = picture->format.mastering.primaries[4];
1280         hdr10.RedPrimary[1]   = picture->format.mastering.primaries[5];
1281         hdr10.WhitePoint[0] = picture->format.mastering.white_point[0];
1282         hdr10.WhitePoint[1] = picture->format.mastering.white_point[1];
1283         hdr10.MinMasteringLuminance = picture->format.mastering.min_luminance;
1284         hdr10.MaxMasteringLuminance = picture->format.mastering.max_luminance;
1285         hdr10.MaxContentLightLevel = picture->format.lighting.MaxCLL;
1286         hdr10.MaxFrameAverageLightLevel = picture->format.lighting.MaxFALL;
1287         IDXGISwapChain4_SetHDRMetaData(sys->dxgiswapChain4, DXGI_HDR_METADATA_TYPE_HDR10, sizeof(hdr10), &hdr10);
1288     }
1289
1290
1291     DXGI_PRESENT_PARAMETERS presentParams;
1292     memset(&presentParams, 0, sizeof(presentParams));
1293     HRESULT hr = IDXGISwapChain1_Present1(sys->dxgiswapChain, 0, 0, &presentParams);
1294     if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
1295     {
1296         /* TODO device lost */
1297         msg_Dbg(vd, "SwapChain Present failed. (hr=0x%lX)", hr);
1298     }
1299
1300     picture_Release(picture);
1301     if (subpicture)
1302         subpicture_Delete(subpicture);
1303
1304     CommonDisplay(vd);
1305 }
1306
1307 static void Direct3D11Destroy(vout_display_t *vd)
1308 {
1309 #if !VLC_WINSTORE_APP
1310     vout_display_sys_t *sys = vd->sys;
1311
1312     if (sys->hd3d11_dll)
1313         FreeLibrary(sys->hd3d11_dll);
1314     if (sys->hd3dcompiler_dll)
1315         FreeLibrary(sys->hd3dcompiler_dll);
1316
1317     sys->OurD3D11CreateDevice = NULL;
1318     sys->OurD3D11CreateDeviceAndSwapChain = NULL;
1319     sys->OurD3DCompile = NULL;
1320     sys->hdxgi_dll = NULL;
1321     sys->hd3d11_dll = NULL;
1322     sys->hd3dcompiler_dll = NULL;
1323 #else
1324     VLC_UNUSED(vd);
1325 #endif
1326 }
1327
1328 #if !VLC_WINSTORE_APP
1329 static HINSTANCE Direct3D11LoadShaderLibrary(void)
1330 {
1331     HINSTANCE instance = NULL;
1332     /* d3dcompiler_47 is the latest on windows 8.1 */
1333     for (int i = 47; i > 41; --i) {
1334         TCHAR filename[19];
1335         _sntprintf(filename, 19, TEXT("D3DCOMPILER_%d.dll"), i);
1336         instance = LoadLibrary(filename);
1337         if (instance) break;
1338     }
1339     return instance;
1340 }
1341 #endif
1342
1343 #define COLOR_RANGE_FULL   1 /* 0-255 */
1344 #define COLOR_RANGE_STUDIO 0 /* 16-235 */
1345
1346 #define TRANSFER_FUNC_10    TRANSFER_FUNC_LINEAR
1347 #define TRANSFER_FUNC_22    TRANSFER_FUNC_SRGB
1348 #define TRANSFER_FUNC_2084  TRANSFER_FUNC_SMPTE_ST2084
1349
1350 #define COLOR_PRIMARIES_BT601  COLOR_PRIMARIES_BT601_525
1351
1352 static const dxgi_color_space color_spaces[] = {
1353 #define DXGIMAP(AXIS, RANGE, GAMMA, SITTING, PRIMARIES) \
1354     { DXGI_COLOR_SPACE_##AXIS##_##RANGE##_G##GAMMA##_##SITTING##_P##PRIMARIES, \
1355       #AXIS " Rec." #PRIMARIES " gamma:" #GAMMA " range:" #RANGE, \
1356       COLOR_AXIS_##AXIS, COLOR_PRIMARIES_BT##PRIMARIES, TRANSFER_FUNC_##GAMMA, \
1357       COLOR_SPACE_BT##PRIMARIES, COLOR_RANGE_##RANGE},
1358
1359     DXGIMAP(RGB,   FULL,     22,    NONE,   709)
1360     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   601)
1361     DXGIMAP(YCBCR, FULL,     22,    LEFT,   601)
1362     DXGIMAP(RGB,   FULL,     10,    NONE,   709)
1363     DXGIMAP(RGB,   STUDIO,   22,    NONE,   709)
1364     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,   709)
1365     DXGIMAP(YCBCR, FULL,     22,    LEFT,   709)
1366     DXGIMAP(RGB,   STUDIO,   22,    NONE,  2020)
1367     DXGIMAP(YCBCR, STUDIO,   22,    LEFT,  2020)
1368     DXGIMAP(YCBCR, FULL,     22,    LEFT,  2020)
1369     DXGIMAP(YCBCR, STUDIO,   22, TOPLEFT,  2020)
1370     DXGIMAP(RGB,   FULL,     22,    NONE,  2020)
1371     DXGIMAP(RGB,   FULL,   2084,    NONE,  2020)
1372     DXGIMAP(YCBCR, STUDIO, 2084,    LEFT,  2020)
1373     DXGIMAP(RGB,   STUDIO, 2084,    NONE,  2020)
1374     DXGIMAP(YCBCR, STUDIO, 2084, TOPLEFT,  2020)
1375     /*DXGIMAP(YCBCR, FULL,     22,    NONE,  2020, 601)*/
1376     {DXGI_COLOR_SPACE_RESERVED, NULL, 0, 0, 0, 0, 0},
1377 #undef DXGIMAP
1378 };
1379
1380 static void D3D11SetColorSpace(vout_display_t *vd)
1381 {
1382     vout_display_sys_t *sys = vd->sys;
1383     HRESULT hr;
1384     int best = -1;
1385     int score, best_score = 0;
1386     UINT support;
1387     IDXGISwapChain3 *dxgiswapChain3 = NULL;
1388     sys->display.colorspace = &color_spaces[0];
1389
1390     hr = ID3D11Device_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain3, (void **)&dxgiswapChain3);
1391     if (FAILED(hr)) {
1392         msg_Warn(vd, "could not get a IDXGISwapChain3");
1393         goto done;
1394     }
1395
1396     bool src_full_range = vd->source.b_color_range_full ||
1397                           /* the YUV->RGB conversion already output full range */
1398                           is_d3d11_opaque(vd->source.i_chroma) ||
1399                           vlc_fourcc_IsYUV(vd->source.i_chroma);
1400 #if VLC_WINSTORE_APP
1401     if (!src_full_range && isXboxHardware(sys->d3ddevice)) {
1402         /* even in full range the Xbox outputs with less range so use the max we can*/
1403         src_full_range = true;
1404     }
1405 #endif
1406
1407     /* pick the best output based on color support and transfer */
1408     /* TODO support YUV output later */
1409     for (int i=0; color_spaces[i].name; ++i)
1410     {
1411         hr = IDXGISwapChain3_CheckColorSpaceSupport(dxgiswapChain3, color_spaces[i].dxgi, &support);
1412         if (SUCCEEDED(hr) && support) {
1413             msg_Dbg(vd, "supports colorspace %s", color_spaces[i].name);
1414             score = 0;
1415             if (color_spaces[i].primaries == vd->source.primaries)
1416                 score++;
1417             if (color_spaces[i].color == vd->source.space)
1418                 score += 2; /* we don't want to translate color spaces */
1419             if (color_spaces[i].transfer == vd->source.transfer ||
1420                 /* favor 2084 output for HLG source */
1421                 (color_spaces[i].transfer == TRANSFER_FUNC_SMPTE_ST2084 && vd->source.transfer == TRANSFER_FUNC_HLG))
1422                 score++;
1423             if (color_spaces[i].b_full_range == src_full_range)
1424                 score++;
1425             if (score > best_score || (score && best == -1)) {
1426                 best = i;
1427                 best_score = score;
1428             }
1429         }
1430     }
1431
1432     if (best == -1)
1433     {
1434         best = 0;
1435         msg_Warn(vd, "no matching colorspace found force %s", color_spaces[best].name);
1436     }
1437     hr = IDXGISwapChain3_SetColorSpace1(dxgiswapChain3, color_spaces[best].dxgi);
1438     if (SUCCEEDED(hr))
1439     {
1440         sys->display.colorspace = &color_spaces[best];
1441         msg_Dbg(vd, "using colorspace %s", sys->display.colorspace->name);
1442     }
1443     else
1444         msg_Err(vd, "Failed to set colorspace %s. (hr=0x%lX)", sys->display.colorspace->name, hr);
1445 done:
1446     /* guestimate the display peak luminance */
1447     switch (sys->display.colorspace->transfer)
1448     {
1449     case TRANSFER_FUNC_LINEAR:
1450     case TRANSFER_FUNC_SRGB:
1451         sys->display.luminance_peak = DEFAULT_BRIGHTNESS;
1452         break;
1453     case TRANSFER_FUNC_SMPTE_ST2084:
1454         sys->display.luminance_peak = 10000;
1455         break;
1456     /* there is no other output transfer on Windows */
1457     default:
1458         vlc_assert_unreachable();
1459     }
1460
1461     if (dxgiswapChain3)
1462         IDXGISwapChain3_Release(dxgiswapChain3);
1463 }
1464
1465 static const d3d_format_t *GetDirectRenderingFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1466 {
1467     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1468     if (is_d3d11_opaque(i_src_chroma))
1469         supportFlags |= D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1470     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1471 }
1472
1473 static const d3d_format_t *GetDirectDecoderFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1474 {
1475     UINT supportFlags = D3D11_FORMAT_SUPPORT_DECODER_OUTPUT;
1476     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, is_d3d11_opaque(i_src_chroma), supportFlags );
1477 }
1478
1479 static const d3d_format_t *GetDisplayFormatByDepth(vout_display_t *vd, uint8_t bit_depth, bool from_processor)
1480 {
1481     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD;
1482     if (from_processor)
1483         supportFlags |= D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
1484     return FindD3D11Format( vd->sys->d3ddevice, 0, bit_depth, false, supportFlags );
1485 }
1486
1487 static const d3d_format_t *GetBlendableFormat(vout_display_t *vd, vlc_fourcc_t i_src_chroma)
1488 {
1489     UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_BLENDABLE;
1490     return FindD3D11Format( vd->sys->d3ddevice, i_src_chroma, 0, false, supportFlags );
1491 }
1492
1493 static int Direct3D11Open(vout_display_t *vd, video_format_t *fmt)
1494 {
1495     vout_display_sys_t *sys = vd->sys;
1496     IDXGIFactory2 *dxgifactory;
1497
1498 #if !VLC_WINSTORE_APP
1499
1500     UINT creationFlags = 0;
1501     HRESULT hr = S_OK;
1502
1503 # if !defined(NDEBUG)
1504 #  if !VLC_WINSTORE_APP
1505     if (IsDebuggerPresent())
1506 #  endif
1507     {
1508         HINSTANCE sdklayer_dll = LoadLibrary(TEXT("d3d11_1sdklayers.dll"));
1509         if (sdklayer_dll) {
1510             creationFlags |= D3D11_CREATE_DEVICE_DEBUG;
1511             FreeLibrary(sdklayer_dll);
1512         }
1513     }
1514 # endif
1515
1516     if (is_d3d11_opaque(fmt->i_chroma))
1517         creationFlags |= D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
1518
1519     DXGI_SWAP_CHAIN_DESC1 scd;
1520     memset(&scd, 0, sizeof(scd));
1521     scd.BufferCount = 2;
1522     scd.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
1523     scd.SampleDesc.Count = 1;
1524     scd.SampleDesc.Quality = 0;
1525     scd.Width = fmt->i_visible_width;
1526     scd.Height = fmt->i_visible_height;
1527     switch(fmt->i_chroma)
1528     {
1529     case VLC_CODEC_D3D11_OPAQUE_10B:
1530         scd.Format = DXGI_FORMAT_R10G10B10A2_UNORM;
1531         break;
1532     default:
1533         scd.Format = DXGI_FORMAT_R8G8B8A8_UNORM; /* TODO: use DXGI_FORMAT_NV12 */
1534         break;
1535     }
1536     //scd.Flags = 512; // DXGI_SWAP_CHAIN_FLAG_YUV_VIDEO;
1537     scd.SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
1538
1539     static const D3D_DRIVER_TYPE driverAttempts[] = {
1540         D3D_DRIVER_TYPE_HARDWARE,
1541         D3D_DRIVER_TYPE_WARP,
1542 #ifndef NDEBUG
1543         D3D_DRIVER_TYPE_REFERENCE,
1544 #endif
1545     };
1546
1547     for (UINT driver = 0; driver < ARRAYSIZE(driverAttempts); driver++) {
1548         D3D_FEATURE_LEVEL i_feature_level;
1549         hr = D3D11CreateDevice(NULL, driverAttempts[driver], NULL, creationFlags,
1550                     NULL, 0, D3D11_SDK_VERSION,
1551                     &sys->d3ddevice, &i_feature_level, &sys->d3dcontext);
1552         if (SUCCEEDED(hr)) {
1553 #ifndef NDEBUG
1554             msg_Dbg(vd, "Created the D3D11 device 0x%p ctx 0x%p type %d level %x.",
1555                     (void *)sys->d3ddevice, (void *)sys->d3dcontext,
1556                     driverAttempts[driver], i_feature_level);
1557 #endif
1558             break;
1559         }
1560     }
1561
1562     if (FAILED(hr)) {
1563        msg_Err(vd, "Could not Create the D3D11 device. (hr=0x%lX)", hr);
1564        return VLC_EGENERIC;
1565     }
1566
1567     IDXGIAdapter *dxgiadapter = D3D11DeviceAdapter(sys->d3ddevice);
1568     if (unlikely(dxgiadapter==NULL)) {
1569        msg_Err(vd, "Could not get the DXGI Adapter");
1570        return VLC_EGENERIC;
1571     }
1572
1573     hr = IDXGIAdapter_GetParent(dxgiadapter, &IID_IDXGIFactory2, (void **)&dxgifactory);
1574     IDXGIAdapter_Release(dxgiadapter);
1575     if (FAILED(hr)) {
1576        msg_Err(vd, "Could not get the DXGI Factory. (hr=0x%lX)", hr);
1577        return VLC_EGENERIC;
1578     }
1579
1580     hr = IDXGIFactory2_CreateSwapChainForHwnd(dxgifactory, (IUnknown *)sys->d3ddevice,
1581                                               sys->sys.hvideownd, &scd, NULL, NULL, &sys->dxgiswapChain);
1582     IDXGIFactory2_Release(dxgifactory);
1583     if (FAILED(hr)) {
1584        msg_Err(vd, "Could not create the SwapChain. (hr=0x%lX)", hr);
1585        return VLC_EGENERIC;
1586     }
1587 #endif
1588
1589     ID3D11Device_QueryInterface( sys->dxgiswapChain, &IID_IDXGISwapChain4, (void **)&sys->dxgiswapChain4);
1590
1591     D3D11SetColorSpace(vd);
1592
1593     // look for the requested pixel format first
1594     sys->picQuadConfig = GetDirectRenderingFormat(vd, fmt->i_chroma);
1595
1596     /* look for a decoder format that can be decoded but not used in shaders */
1597     const d3d_format_t *decoder_format = NULL;
1598     if ( !sys->picQuadConfig && is_d3d11_opaque(fmt->i_chroma) )
1599         decoder_format = GetDirectDecoderFormat(vd, fmt->i_chroma);
1600
1601     // look for any pixel format that we can handle with enough pixels per channel
1602     if ( !sys->picQuadConfig )
1603     {
1604         uint8_t bits_per_channel;
1605         switch (fmt->i_chroma)
1606         {
1607         case VLC_CODEC_D3D11_OPAQUE:
1608             bits_per_channel = 8;
1609             break;
1610         case VLC_CODEC_D3D11_OPAQUE_10B:
1611             bits_per_channel = 10;
1612             break;
1613         default:
1614             {
1615                 const vlc_chroma_description_t *p_format = vlc_fourcc_GetChromaDescription(fmt->i_chroma);
1616                 bits_per_channel = p_format == NULL || p_format->pixel_bits == 0 ? 8 : p_format->pixel_bits;
1617             }
1618             break;
1619         }
1620
1621         sys->picQuadConfig = GetDisplayFormatByDepth(vd, bits_per_channel, decoder_format!=NULL);
1622     }
1623
1624     // look for any pixel format that we can handle
1625     if ( !sys->picQuadConfig )
1626         sys->picQuadConfig = GetDisplayFormatByDepth(vd, 0, false);
1627
1628     if ( !sys->picQuadConfig )
1629     {
1630        msg_Err(vd, "Could not get a suitable texture pixel format");
1631        return VLC_EGENERIC;
1632     }
1633
1634     /* we will need a converter from 420_OPAQUE to our display format */
1635     if ( decoder_format )
1636         sys->decoderFormat = decoder_format->formatTexture;
1637     fmt->i_chroma = sys->picQuadConfig->fourcc;
1638
1639     msg_Dbg( vd, "Using pixel format %s for chroma %4.4s", sys->picQuadConfig->name,
1640                  (char *)&fmt->i_chroma );
1641     DxgiFormatMask( sys->picQuadConfig->formatTexture, fmt );
1642
1643     /* check the region pixel format */
1644     sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_RGBA);
1645     if (!sys->d3dregion_format)
1646         sys->d3dregion_format = GetBlendableFormat(vd, VLC_CODEC_BGRA);
1647
1648     UpdateRects(vd, NULL, NULL, true);
1649
1650 #if defined(HAVE_ID3D11VIDEODECODER)
1651     if( sys->context_lock != INVALID_HANDLE_VALUE )
1652     {
1653         WaitForSingleObjectEx( sys->context_lock, INFINITE, FALSE );
1654     }
1655 #endif
1656     if (Direct3D11CreateResources(vd, fmt)) {
1657 #if defined(HAVE_ID3D11VIDEODECODER)
1658         if( sys->context_lock != INVALID_HANDLE_VALUE )
1659         {
1660             ReleaseMutex( sys->context_lock );
1661         }
1662 #endif
1663         msg_Err(vd, "Failed to allocate resources");
1664         Direct3D11DestroyResources(vd);
1665         return VLC_EGENERIC;
1666     }
1667 #if defined(HAVE_ID3D11VIDEODECODER)
1668     if( sys->context_lock != INVALID_HANDLE_VALUE )
1669     {
1670         ReleaseMutex( sys->context_lock );
1671     }
1672 #endif
1673
1674 #if !VLC_WINSTORE_APP
1675     EventThreadUpdateTitle(sys->sys.event, VOUT_TITLE " (Direct3D11 output)");
1676 #endif
1677
1678     msg_Dbg(vd, "Direct3D11 device adapter successfully initialized");
1679     return VLC_SUCCESS;
1680 }
1681
1682 static void Direct3D11Close(vout_display_t *vd)
1683 {
1684     vout_display_sys_t *sys = vd->sys;
1685
1686     Direct3D11DestroyResources(vd);
1687     if (sys->d3dcontext)
1688     {
1689         ID3D11DeviceContext_Flush(sys->d3dcontext);
1690         ID3D11DeviceContext_Release(sys->d3dcontext);
1691         sys->d3dcontext = NULL;
1692     }
1693     if (sys->d3ddevice)
1694     {
1695         ID3D11Device_Release(sys->d3ddevice);
1696         sys->d3ddevice = NULL;
1697     }
1698     if (sys->dxgiswapChain4)
1699     {
1700         IDXGISwapChain_Release(sys->dxgiswapChain4);
1701         sys->dxgiswapChain4 = NULL;
1702     }
1703     if (sys->dxgiswapChain)
1704     {
1705         IDXGISwapChain_Release(sys->dxgiswapChain);
1706         sys->dxgiswapChain = NULL;
1707     }
1708
1709     msg_Dbg(vd, "Direct3D11 device adapter closed");
1710 }
1711
1712 static void UpdatePicQuadPosition(vout_display_t *vd)
1713 {
1714     vout_display_sys_t *sys = vd->sys;
1715
1716     sys->picQuad.cropViewport.Width    = RECTWidth(sys->sys.rect_dest_clipped);
1717     sys->picQuad.cropViewport.Height   = RECTHeight(sys->sys.rect_dest_clipped);
1718     sys->picQuad.cropViewport.TopLeftX = sys->sys.rect_dest_clipped.left;
1719     sys->picQuad.cropViewport.TopLeftY = sys->sys.rect_dest_clipped.top;
1720
1721     sys->picQuad.cropViewport.MinDepth = 0.0f;
1722     sys->picQuad.cropViewport.MaxDepth = 1.0f;
1723
1724     SetQuadVSProjection(vd, &sys->picQuad, &vd->cfg->viewpoint);
1725
1726 #ifndef NDEBUG
1727     msg_Dbg(vd, "picQuad position (%.02f,%.02f) %.02fx%.02f", sys->picQuad.cropViewport.TopLeftX, sys->picQuad.cropViewport.TopLeftY, sys->picQuad.cropViewport.Width, sys->picQuad.cropViewport.Height );
1728 #endif
1729 }
1730
1731 static ID3DBlob* CompileShader(vout_display_t *vd, const char *psz_shader, bool pixel)
1732 {
1733     vout_display_sys_t *sys = vd->sys;
1734     ID3DBlob* pShaderBlob = NULL, *pErrBlob;
1735
1736     /* TODO : Match the version to the D3D_FEATURE_LEVEL */
1737     HRESULT hr = D3DCompile(psz_shader, strlen(psz_shader),
1738                             NULL, NULL, NULL, "main",
1739                             pixel ? (sys->legacy_shader ? "ps_4_0_level_9_1" : "ps_4_0") :
1740                                     (sys->legacy_shader ? "vs_4_0_level_9_1" : "vs_4_0"),
1741                             0, 0, &pShaderBlob, &pErrBlob);
1742
1743     if (FAILED(hr)) {
1744         char *err = pErrBlob ? ID3D10Blob_GetBufferPointer(pErrBlob) : NULL;
1745         msg_Err(vd, "invalid %s Shader (hr=0x%lX): %s", pixel?"Pixel":"Vertex", hr, err );
1746         if (pErrBlob)
1747             ID3D10Blob_Release(pErrBlob);
1748         return NULL;
1749     }
1750     return pShaderBlob;
1751 }
1752
1753 static bool IsRGBShader(const d3d_format_t *cfg)
1754 {
1755     return cfg->resourceFormat[0] != DXGI_FORMAT_R8_UNORM &&
1756            cfg->resourceFormat[0] != DXGI_FORMAT_R16_UNORM &&
1757            cfg->formatTexture != DXGI_FORMAT_YUY2;
1758 }
1759
1760 static HRESULT CompilePixelShader(vout_display_t *vd, const d3d_format_t *format,
1761                                   video_transfer_func_t transfer, bool src_full_range,
1762                                   ID3D11PixelShader **output)
1763 {
1764     vout_display_sys_t *sys = vd->sys;
1765
1766     static const char *DEFAULT_NOOP = "return rgb";
1767     const char *psz_sampler;
1768     const char *psz_src_transform     = DEFAULT_NOOP;
1769     const char *psz_display_transform = DEFAULT_NOOP;
1770     const char *psz_tone_mapping      = DEFAULT_NOOP;
1771     const char *psz_peak_luminance    = DEFAULT_NOOP;
1772     const char *psz_adjust_range      = DEFAULT_NOOP;
1773     char *psz_luminance = NULL;
1774     char *psz_range = NULL;
1775
1776     switch (format->formatTexture)
1777     {
1778     case DXGI_FORMAT_NV12:
1779     case DXGI_FORMAT_P010:
1780         psz_sampler =
1781                 "sample.x  = shaderTexture[0].Sample(SampleType, In.Texture).x;\
1782                 sample.yz = shaderTexture[1].Sample(SampleType, In.Texture).xy;\
1783                 sample.a  = 1;";
1784         break;
1785     case DXGI_FORMAT_YUY2:
1786         psz_sampler =
1787                 "sample.x  = shaderTexture[0].Sample(SampleType, In.Texture).x;\
1788                 sample.y  = shaderTexture[0].Sample(SampleType, In.Texture).y;\
1789                 sample.z  = shaderTexture[0].Sample(SampleType, In.Texture).a;\
1790                 sample.a  = 1;";
1791         break;
1792     case DXGI_FORMAT_R8G8B8A8_UNORM:
1793     case DXGI_FORMAT_B8G8R8A8_UNORM:
1794     case DXGI_FORMAT_B5G6R5_UNORM:
1795         psz_sampler =
1796                 "sample = shaderTexture[0].Sample(SampleType, In.Texture);";
1797         break;
1798     default:
1799         vlc_assert_unreachable();
1800     }
1801
1802     video_transfer_func_t src_transfer;
1803     unsigned src_luminance_peak;
1804     switch (transfer)
1805     {
1806         case TRANSFER_FUNC_SMPTE_ST2084:
1807             /* TODO ajust this value using HDR metadata ? */
1808             src_luminance_peak = 7500;
1809             break;
1810         case TRANSFER_FUNC_HLG:
1811             src_luminance_peak = 1000;
1812             break;
1813         case TRANSFER_FUNC_BT709:
1814         case TRANSFER_FUNC_SRGB:
1815             src_luminance_peak = DEFAULT_BRIGHTNESS;
1816             break;
1817         default:
1818             msg_Dbg(vd, "unhandled source transfer %d", transfer);
1819             src_luminance_peak = DEFAULT_BRIGHTNESS;
1820             break;
1821     }
1822
1823     if (transfer != sys->display.colorspace->transfer)
1824     {
1825         /* we need to go in linear mode */
1826         switch (transfer)
1827         {
1828             case TRANSFER_FUNC_SMPTE_ST2084:
1829                 /* ST2084 to Linear */
1830                 psz_src_transform =
1831                        ST2084_PQ_CONSTANTS
1832                        "rgb = pow(rgb, 1.0/ST2084_m2);\
1833                         rgb = max(rgb - ST2084_c1, 0.0) / (ST2084_c2 - ST2084_c3 * rgb);\
1834                         rgb = pow(rgb, 1.0/ST2084_m1);\
1835                         return rgb";
1836                 src_transfer = TRANSFER_FUNC_LINEAR;
1837                 break;
1838             case TRANSFER_FUNC_HLG:
1839                 /* HLG to Linear */
1840                 psz_src_transform =
1841                        "rgb.r = inverse_HLG(rgb.r);\
1842                         rgb.g = inverse_HLG(rgb.g);\
1843                         rgb.b = inverse_HLG(rgb.b);\
1844                         return rgb / 20.0";
1845                 src_transfer = TRANSFER_FUNC_LINEAR;
1846                 break;
1847             case TRANSFER_FUNC_BT709:
1848                 psz_src_transform = "return pow(rgb, 1.0 / 0.45)";
1849                 src_transfer = TRANSFER_FUNC_LINEAR;
1850                 break;
1851             case TRANSFER_FUNC_SRGB:
1852                 psz_src_transform = "return pow(rgb, 2.2)";
1853                 src_transfer = TRANSFER_FUNC_LINEAR;
1854                 break;
1855             default:
1856                 msg_Dbg(vd, "unhandled source transfer %d", transfer);
1857                 src_transfer = transfer;
1858                 break;
1859         }
1860
1861         switch (sys->display.colorspace->transfer)
1862         {
1863             case TRANSFER_FUNC_SRGB:
1864                 if (src_transfer == TRANSFER_FUNC_LINEAR)
1865                 {
1866                     /* Linear to sRGB */
1867                     psz_display_transform = "return pow(rgb, 1.0 / 2.2)";
1868
1869                     if (transfer == TRANSFER_FUNC_SMPTE_ST2084 || transfer == TRANSFER_FUNC_HLG)
1870                     {
1871                         /* HDR tone mapping */
1872                         psz_tone_mapping =
1873                             "static const float3 HABLE_DIV = hable(11.2);\
1874                             rgb = hable(rgb) / HABLE_DIV;\
1875                             return rgb";
1876                     }
1877                 }
1878                 else
1879                     msg_Warn(vd, "don't know how to transfer from %d to sRGB", src_transfer);
1880                 break;
1881
1882             case TRANSFER_FUNC_SMPTE_ST2084:
1883                 if (src_transfer == TRANSFER_FUNC_LINEAR)
1884                 {
1885                     /* Linear to ST2084 */
1886                     psz_display_transform =
1887                            ST2084_PQ_CONSTANTS
1888                            "rgb = pow(rgb, ST2084_m1);\
1889                             rgb = (ST2084_c1 + ST2084_c2 * rgb) / (1 + ST2084_c3 * rgb);\
1890                             rgb = pow(rgb, ST2084_m2);\
1891                             return rgb";
1892                 }
1893                 else
1894                     msg_Warn(vd, "don't know how to transfer from %d to SMPTE ST 2084", src_transfer);
1895                 break;
1896             default:
1897                 msg_Warn(vd, "don't know how to transfer from %d to %d", src_transfer, sys->display.colorspace->transfer);
1898                 break;
1899         }
1900     }
1901
1902     if (src_luminance_peak != sys->display.luminance_peak)
1903     {
1904         psz_luminance = malloc(64);
1905         if (likely(psz_luminance))
1906         {
1907             sprintf(psz_luminance, "return rgb * %f", (float)src_luminance_peak / (float)sys->display.luminance_peak);
1908             psz_peak_luminance = psz_luminance;
1909         }
1910     }
1911
1912     int range_adjust = sys->display.colorspace->b_full_range;
1913 #if VLC_WINSTORE_APP
1914     if (isXboxHardware(sys->d3ddevice)) {
1915         /* the Xbox lies, when it says it outputs full range it's less than full range */
1916         range_adjust--;
1917     }
1918 #endif
1919     if (!IsRGBShader(format))
1920         range_adjust--; /* the YUV->RGB conversion already output full range */
1921     if (src_full_range)
1922         range_adjust--;
1923
1924     if (range_adjust != 0)
1925     {
1926         psz_range = malloc(256);
1927         if (likely(psz_range))
1928         {
1929             FLOAT itu_black_level;
1930             FLOAT itu_range_factor;
1931             FLOAT itu_white_level;
1932             switch (format->bitsPerChannel)
1933             {
1934             case 8:
1935                 /* Rec. ITU-R BT.709-6 §4.6 */
1936                 itu_black_level  =              16.f / 255.f;
1937                 itu_white_level  =             235.f / 255.f;
1938                 itu_range_factor = (float)(235 - 16) / 255.f;
1939                 break;
1940             case 10:
1941                 /* Rec. ITU-R BT.709-6 §4.6 */
1942                 itu_black_level  =              64.f / 1023.f;
1943                 itu_white_level  =             940.f / 1023.f;
1944                 itu_range_factor = (float)(940 - 64) / 1023.f;
1945                 break;
1946             case 12:
1947                 /* Rec. ITU-R BT.2020-2 Table 5 */
1948                 itu_black_level  =               256.f / 4095.f;
1949                 itu_white_level  =              3760.f / 4095.f;
1950                 itu_range_factor = (float)(3760 - 256) / 4095.f;
1951                 break;
1952             default:
1953                 /* unknown bitdepth, use approximation for infinite bit depth */
1954                 itu_black_level  =              16.f / 256.f;
1955                 itu_white_level  =             235.f / 256.f;
1956                 itu_range_factor = (float)(235 - 16) / 256.f;
1957                 break;
1958             }
1959
1960             FLOAT black_level = 0;
1961             FLOAT range_factor = 1.0f;
1962             if (range_adjust > 0)
1963             {
1964                 /* expand the range from studio to full range */
1965                 while (range_adjust--)
1966                 {
1967                     black_level -= itu_black_level;
1968                     range_factor /= itu_range_factor;
1969                 }
1970                 sprintf(psz_range, "return max(0,min(1,(rgb + %f) * %f))",
1971                         black_level, range_factor);
1972             }
1973             else
1974             {
1975                 /* shrink the range to studio range */
1976                 while (range_adjust++)
1977                 {
1978                     black_level += itu_black_level;
1979                     range_factor *= itu_range_factor;
1980                 }
1981                 sprintf(psz_range, "return clamp(rgb + %f * %f,%f,%f)",
1982                         black_level, range_factor, itu_black_level, itu_white_level);
1983             }
1984             psz_adjust_range = psz_range;
1985         }
1986     }
1987
1988     char *shader = malloc(strlen(globPixelShaderDefault) + 32 + strlen(psz_sampler) +
1989                           strlen(psz_src_transform) + strlen(psz_display_transform) +
1990                           strlen(psz_tone_mapping) + strlen(psz_peak_luminance) + strlen(psz_adjust_range));
1991     if (!shader)
1992     {
1993         msg_Err(vd, "no room for the Pixel Shader");
1994         free(psz_luminance);
1995         free(psz_range);
1996         return E_OUTOFMEMORY;
1997     }
1998     sprintf(shader, globPixelShaderDefault, sys->legacy_shader ? "" : "Array", psz_src_transform,
1999             psz_display_transform, psz_tone_mapping, psz_peak_luminance, psz_adjust_range, psz_sampler);
2000 #ifndef NDEBUG
2001     if (!IsRGBShader(format)) {
2002         msg_Dbg(vd,"psz_src_transform %s", psz_src_transform);
2003         msg_Dbg(vd,"psz_tone_mapping %s", psz_tone_mapping);
2004         msg_Dbg(vd,"psz_peak_luminance %s", psz_peak_luminance);
2005         msg_Dbg(vd,"psz_display_transform %s", psz_display_transform);
2006         msg_Dbg(vd,"psz_adjust_range %s", psz_adjust_range);
2007     }
2008 #endif
2009     free(psz_luminance);
2010     free(psz_range);
2011
2012     ID3DBlob *pPSBlob = CompileShader(vd, shader, true);
2013     free(shader);
2014     if (!pPSBlob)
2015         return E_INVALIDARG;
2016
2017     HRESULT hr = ID3D11Device_CreatePixelShader(sys->d3ddevice,
2018                                                 (void *)ID3D10Blob_GetBufferPointer(pPSBlob),
2019                                                 ID3D10Blob_GetBufferSize(pPSBlob), NULL, output);
2020
2021     ID3D10Blob_Release(pPSBlob);
2022     return hr;
2023 }
2024
2025 #if (!VLC_WINSTORE_APP) && defined (HAVE_ID3D11VIDEODECODER)
2026
2027 static HKEY GetAdapterRegistry(DXGI_ADAPTER_DESC *adapterDesc)
2028 {
2029     HKEY hKey;
2030     TCHAR key[128];
2031     TCHAR szData[256], lookup[256];
2032     DWORD len = 256;
2033
2034     _sntprintf(lookup, 256, TEXT("pci\\ven_%04x&dev_%04x&rev_cf"), adapterDesc->VendorId, adapterDesc->DeviceId);
2035     for (int i=0;;i++)
2036     {
2037         _sntprintf(key, 128, TEXT("SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\%04d"), i);
2038         if( RegOpenKeyEx(HKEY_LOCAL_MACHINE, key, 0, KEY_READ, &hKey) != ERROR_SUCCESS )
2039             return NULL;
2040
2041         if( RegQueryValueEx( hKey, TEXT("MatchingDeviceId"), NULL, NULL, (LPBYTE) &szData, &len ) == ERROR_SUCCESS ) {
2042             if (_tcscmp(lookup, szData) == 0)
2043                 return hKey;
2044         }
2045
2046         RegCloseKey(hKey);
2047     }
2048     return NULL;
2049 }
2050 #endif
2051
2052 static bool CanUseTextureArray(vout_display_t *vd)
2053 {
2054 #ifndef HAVE_ID3D11VIDEODECODER
2055     (void) vd;
2056     return false;
2057 #else
2058     vout_display_sys_t *sys = vd->sys;
2059     TCHAR szData[256];
2060     DWORD len = 256;
2061     IDXGIAdapter *pAdapter = D3D11DeviceAdapter(sys->d3ddevice);
2062     if (!pAdapter)
2063         return false;
2064
2065     DXGI_ADAPTER_DESC adapterDesc;
2066     HRESULT hr = IDXGIAdapter_GetDesc(pAdapter, &adapterDesc);
2067     IDXGIAdapter_Release(pAdapter);
2068     if (FAILED(hr))
2069         return false;
2070
2071     /* ATI/AMD hardware has issues with pixel shaders with Texture2DArray */
2072     if (adapterDesc.VendorId != 0x1002)
2073         return true;
2074
2075 #if !VLC_WINSTORE_APP
2076     HKEY hKey = GetAdapterRegistry(&adapterDesc);
2077     if (hKey == NULL)
2078         return false;
2079
2080     LONG err = RegQueryValueEx( hKey, TEXT("DriverVersion"), NULL, NULL, (LPBYTE) &szData, &len );
2081     RegCloseKey(hKey);
2082
2083     if (err == ERROR_SUCCESS )
2084     {
2085         int wddm, d3d_features, revision, build;
2086         /* see https://msdn.microsoft.com/windows/hardware/commercialize/design/compatibility/device-graphics */
2087         if (_stscanf(szData, TEXT("%d.%d.%d.%d"), &wddm, &d3d_features, &revision, &build) == 4)
2088             /* it should be OK starting from driver 22.19.128.xxx */
2089             return wddm > 22 || (wddm == 22 && (d3d_features > 19 || (d3d_features == 19 && revision >= 128)));
2090     }
2091 #endif
2092     msg_Dbg(vd, "fallback to legacy shader mode for old AMD drivers");
2093     return false;
2094 #endif
2095 }
2096
2097 /* TODO : handle errors better
2098    TODO : seperate out into smaller functions like createshaders */
2099 static int Direct3D11CreateResources(vout_display_t *vd, video_format_t *fmt)
2100 {
2101     vout_display_sys_t *sys = vd->sys;
2102     HRESULT hr;
2103
2104 #if defined(HAVE_ID3D11VIDEODECODER)
2105     sys->context_lock = CreateMutexEx( NULL, NULL, 0, SYNCHRONIZE );
2106     ID3D11Device_SetPrivateData( sys->d3ddevice, &GUID_CONTEXT_MUTEX, sizeof( sys->context_lock ), &sys->context_lock );
2107 #endif
2108
2109     hr = UpdateBackBuffer(vd);
2110     if (FAILED(hr)) {
2111        msg_Err(vd, "Could not update the backbuffer. (hr=0x%lX)", hr);
2112        return VLC_EGENERIC;
2113     }
2114
2115     ID3D11BlendState *pSpuBlendState;
2116     D3D11_BLEND_DESC spuBlendDesc = { 0 };
2117     spuBlendDesc.RenderTarget[0].BlendEnable = TRUE;
2118     spuBlendDesc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA;
2119     spuBlendDesc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA;
2120     spuBlendDesc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD;
2121
2122     spuBlendDesc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE;
2123     spuBlendDesc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO;
2124     spuBlendDesc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;
2125
2126     spuBlendDesc.RenderTarget[0].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
2127
2128     spuBlendDesc.RenderTarget[1].BlendEnable = TRUE;
2129     spuBlendDesc.RenderTarget[1].SrcBlend = D3D11_BLEND_ONE;
2130     spuBlendDesc.RenderTarget[1].DestBlend = D3D11_BLEND_ZERO;
2131     spuBlendDesc.RenderTarget[1].BlendOp = D3D11_BLEND_OP_ADD;
2132
2133     spuBlendDesc.RenderTarget[1].SrcBlendAlpha = D3D11_BLEND_ONE;
2134     spuBlendDesc.RenderTarget[1].DestBlendAlpha = D3D11_BLEND_ZERO;
2135     spuBlendDesc.RenderTarget[1].BlendOpAlpha = D3D11_BLEND_OP_ADD;
2136
2137     spuBlendDesc.RenderTarget[1].RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
2138     hr = ID3D11Device_CreateBlendState(sys->d3ddevice, &spuBlendDesc, &pSpuBlendState);
2139     if (FAILED(hr)) {
2140        msg_Err(vd, "Could not create SPU blend state. (hr=0x%lX)", hr);
2141        return VLC_EGENERIC;
2142     }
2143     ID3D11DeviceContext_OMSetBlendState(sys->d3dcontext, pSpuBlendState, NULL, 0xFFFFFFFF);
2144     ID3D11BlendState_Release(pSpuBlendState);
2145
2146     /* disable depth testing as we're only doing 2D
2147      * see https://msdn.microsoft.com/en-us/library/windows/desktop/bb205074%28v=vs.85%29.aspx
2148      * see http://rastertek.com/dx11tut11.html
2149     */
2150     D3D11_DEPTH_STENCIL_DESC stencilDesc;
2151     ZeroMemory(&stencilDesc, sizeof(stencilDesc));
2152
2153     ID3D11DepthStencilState *pDepthStencilState;
2154     hr = ID3D11Device_CreateDepthStencilState(sys->d3ddevice, &stencilDesc, &pDepthStencilState );
2155     if (SUCCEEDED(hr)) {
2156         ID3D11DeviceContext_OMSetDepthStencilState(sys->d3dcontext, pDepthStencilState, 0);
2157         ID3D11DepthStencilState_Release(pDepthStencilState);
2158     }
2159
2160     sys->legacy_shader = !CanUseTextureArray(vd);
2161     vd->info.is_slow = !is_d3d11_opaque(fmt->i_chroma);
2162
2163     hr = CompilePixelShader(vd, sys->picQuadConfig, fmt->transfer, fmt->b_color_range_full, &sys->picQuadPixelShader);
2164     if (FAILED(hr))
2165     {
2166 #ifdef HAVE_ID3D11VIDEODECODER
2167         if (!sys->legacy_shader)
2168         {
2169             sys->legacy_shader = true;
2170             msg_Dbg(vd, "fallback to legacy shader mode");
2171             hr = CompilePixelShader(vd, sys->picQuadConfig, fmt->transfer, fmt->b_color_range_full, &sys->picQuadPixelShader);
2172         }
2173 #endif
2174         if (FAILED(hr))
2175         {
2176             msg_Err(vd, "Failed to create the pixel shader. (hr=0x%lX)", hr);
2177             return VLC_EGENERIC;
2178         }
2179     }
2180
2181     if (sys->d3dregion_format != NULL)
2182     {
2183         hr = CompilePixelShader(vd, sys->d3dregion_format, TRANSFER_FUNC_SRGB, true, &sys->pSPUPixelShader);
2184         if (FAILED(hr))
2185         {
2186             ID3D11PixelShader_Release(sys->picQuadPixelShader);
2187             sys->picQuadPixelShader = NULL;
2188             msg_Err(vd, "Failed to create the SPU pixel shader. (hr=0x%lX)", hr);
2189             return VLC_EGENERIC;
2190         }
2191     }
2192
2193     ID3DBlob *pVSBlob = CompileShader(vd, globVertexShaderFlat , false);
2194     if (!pVSBlob)
2195         return VLC_EGENERIC;
2196
2197     hr = ID3D11Device_CreateVertexShader(sys->d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2198                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->flatVSShader);
2199
2200     if(FAILED(hr)) {
2201       ID3D11Device_Release(pVSBlob);
2202       msg_Err(vd, "Failed to create the flat vertex shader. (hr=0x%lX)", hr);
2203       return VLC_EGENERIC;
2204     }
2205
2206     D3D11_INPUT_ELEMENT_DESC layout[] = {
2207     { "POSITION", 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
2208     { "TEXCOORD", 0, DXGI_FORMAT_R32G32_FLOAT,    0, D3D11_APPEND_ALIGNED_ELEMENT, D3D11_INPUT_PER_VERTEX_DATA, 0},
2209     };
2210
2211     ID3D11InputLayout* pVertexLayout = NULL;
2212     hr = ID3D11Device_CreateInputLayout(sys->d3ddevice, layout, 2, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2213                                         ID3D10Blob_GetBufferSize(pVSBlob), &pVertexLayout);
2214
2215     ID3D10Blob_Release(pVSBlob);
2216
2217     if(FAILED(hr)) {
2218       msg_Err(vd, "Failed to create the vertex input layout. (hr=0x%lX)", hr);
2219       return VLC_EGENERIC;
2220     }
2221     ID3D11DeviceContext_IASetInputLayout(sys->d3dcontext, pVertexLayout);
2222     ID3D11InputLayout_Release(pVertexLayout);
2223
2224     pVSBlob = CompileShader(vd, globVertexShaderProjection, false);
2225     if (!pVSBlob)
2226         return VLC_EGENERIC;
2227
2228     hr = ID3D11Device_CreateVertexShader(sys->d3ddevice, (void *)ID3D10Blob_GetBufferPointer(pVSBlob),
2229                                         ID3D10Blob_GetBufferSize(pVSBlob), NULL, &sys->projectionVSShader);
2230
2231     if(FAILED(hr)) {
2232       ID3D11Device_Release(pVSBlob);
2233       msg_Err(vd, "Failed to create the projection vertex shader. (hr=0x%lX)", hr);
2234       return VLC_EGENERIC;
2235     }
2236     ID3D10Blob_Release(pVSBlob);
2237
2238     ID3D11DeviceContext_IASetPrimitiveTopology(sys->d3dcontext, D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
2239
2240     UpdatePicQuadPosition(vd);
2241
2242     D3D11_SAMPLER_DESC sampDesc;
2243     memset(&sampDesc, 0, sizeof(sampDesc));
2244     sampDesc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT;
2245     sampDesc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP;
2246     sampDesc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP;
2247     sampDesc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP;
2248     sampDesc.ComparisonFunc = D3D11_COMPARISON_ALWAYS;
2249     sampDesc.MinLOD = 0;
2250     sampDesc.MaxLOD = D3D11_FLOAT32_MAX;
2251
2252     ID3D11SamplerState *d3dsampState;
2253     hr = ID3D11Device_CreateSamplerState(sys->d3ddevice, &sampDesc, &d3dsampState);
2254
2255     if (FAILED(hr)) {
2256       msg_Err(vd, "Could not Create the D3d11 Sampler State. (hr=0x%lX)", hr);
2257       return VLC_EGENERIC;
2258     }
2259     ID3D11DeviceContext_PSSetSamplers(sys->d3dcontext, 0, 1, &d3dsampState);
2260     ID3D11SamplerState_Release(d3dsampState);
2261
2262     msg_Dbg(vd, "Direct3D11 resources created");
2263     return VLC_SUCCESS;
2264 }
2265
2266 static void Direct3D11DestroyPool(vout_display_t *vd)
2267 {
2268     vout_display_sys_t *sys = vd->sys;
2269
2270     if (sys->sys.pool)
2271         picture_pool_Release(sys->sys.pool);
2272     sys->sys.pool = NULL;
2273 }
2274
2275 /**
2276  * Compute the vertex ordering needed to rotate the video. Without
2277  * rotation, the vertices of the rectangle are defined in a clockwise
2278  * order. This function computes a remapping of the coordinates to
2279  * implement the rotation, given fixed texture coordinates.
2280  * The unrotated order is the following:
2281  * 0--1
2282  * |  |
2283  * 3--2
2284  * For a 180 degrees rotation it should like this:
2285  * 2--3
2286  * |  |
2287  * 1--0
2288  * Vertex 0 should be assigned coordinates at index 2 from the
2289  * unrotated order and so on, thus yielding order: 2 3 0 1.
2290  */
2291 static void orientationVertexOrder(video_orientation_t orientation, int vertex_order[static 4])
2292 {
2293     switch (orientation) {
2294         case ORIENT_ROTATED_90:
2295             vertex_order[0] = 3;
2296             vertex_order[1] = 0;
2297             vertex_order[2] = 1;
2298             vertex_order[3] = 2;
2299             break;
2300         case ORIENT_ROTATED_270:
2301             vertex_order[0] = 1;
2302             vertex_order[1] = 2;
2303             vertex_order[2] = 3;
2304             vertex_order[3] = 0;
2305             break;
2306         case ORIENT_ROTATED_180:
2307             vertex_order[0] = 2;
2308             vertex_order[1] = 3;
2309             vertex_order[2] = 0;
2310             vertex_order[3] = 1;
2311             break;
2312         case ORIENT_TRANSPOSED:
2313             vertex_order[0] = 2;
2314             vertex_order[1] = 1;
2315             vertex_order[2] = 0;
2316             vertex_order[3] = 3;
2317             break;
2318         case ORIENT_HFLIPPED:
2319             vertex_order[0] = 3;
2320             vertex_order[1] = 2;
2321             vertex_order[2] = 1;
2322             vertex_order[3] = 0;
2323             break;
2324         case ORIENT_VFLIPPED:
2325             vertex_order[0] = 1;
2326             vertex_order[1] = 0;
2327             vertex_order[2] = 3;
2328             vertex_order[3] = 2;
2329             break;
2330         case ORIENT_ANTI_TRANSPOSED: /* transpose + vflip */
2331             vertex_order[0] = 1;
2332             vertex_order[1] = 2;
2333             vertex_order[2] = 3;
2334             vertex_order[3] = 0;
2335             break;
2336        default:
2337             vertex_order[0] = 0;
2338             vertex_order[1] = 1;
2339             vertex_order[2] = 2;
2340             vertex_order[3] = 3;
2341             break;
2342     }
2343 }
2344
2345 static void SetupQuadFlat(d3d_vertex_t *dst_data, const RECT *output,
2346                           const d3d_quad_t *quad,
2347                           WORD *triangle_pos, video_orientation_t orientation)
2348 {
2349     unsigned int dst_x = output->left;
2350     unsigned int dst_y = output->top;
2351     unsigned int dst_width = RECTWidth(*output);
2352     unsigned int dst_height = RECTHeight(*output);
2353     unsigned int src_width = quad->i_width;
2354     unsigned int src_height = quad->i_height;
2355     LONG MidY = output->top + output->bottom; // /2
2356     LONG MidX = output->left + output->right; // /2
2357
2358     /* the clamping will not work properly on the side of the texture as it
2359      * will have decoder pixels not mean to be displayed but used for interpolation
2360      * So we lose the last line that will be partially green */
2361     if (src_width != dst_width)
2362         src_width += 1;
2363     if (src_height != dst_height)
2364         src_height += 1;
2365
2366     float top, bottom, left, right;
2367     top    =  (float)MidY / (float)(MidY - 2*dst_y);
2368     bottom = -(2.f*src_height - MidY) / (float)(MidY - 2*dst_y);
2369     right  =  (2.f*src_width  - MidX) / (float)(MidX - 2*dst_x);
2370     left   = -(float)MidX / (float)(MidX - 2*dst_x);
2371
2372     const float vertices_coords[4][2] = {
2373         { left,  bottom },
2374         { right, bottom },
2375         { right, top    },
2376         { left,  top    },
2377     };
2378
2379     /* Compute index remapping necessary to implement the rotation. */
2380     int vertex_order[4];
2381     orientationVertexOrder(orientation, vertex_order);
2382
2383     for (int i = 0; i < 4; ++i) {
2384         dst_data[i].position.x  = vertices_coords[vertex_order[i]][0];
2385         dst_data[i].position.y  = vertices_coords[vertex_order[i]][1];
2386     }
2387
2388     // bottom left
2389     dst_data[0].position.z = 0.0f;
2390     dst_data[0].texture.x = 0.0f;
2391     dst_data[0].texture.y = 1.0f;
2392
2393     // bottom right
2394     dst_data[1].position.z = 0.0f;
2395     dst_data[1].texture.x = 1.0f;
2396     dst_data[1].texture.y = 1.0f;
2397
2398     // top right
2399     dst_data[2].position.z = 0.0f;
2400     dst_data[2].texture.x = 1.0f;
2401     dst_data[2].texture.y = 0.0f;
2402
2403     // top left
2404     dst_data[3].position.z = 0.0f;
2405     dst_data[3].texture.x = 0.0f;
2406     dst_data[3].texture.y = 0.0f;
2407
2408     triangle_pos[0] = 3;
2409     triangle_pos[1] = 1;
2410     triangle_pos[2] = 0;
2411
2412     triangle_pos[3] = 2;
2413     triangle_pos[4] = 1;
2414     triangle_pos[5] = 3;
2415 }
2416
2417 #define SPHERE_SLICES 128
2418 #define nbLatBands SPHERE_SLICES
2419 #define nbLonBands SPHERE_SLICES
2420
2421 static void SetupQuadSphere(d3d_vertex_t *dst_data, WORD *triangle_pos)
2422 {
2423     for (unsigned lat = 0; lat <= nbLatBands; lat++) {
2424         float theta = lat * (float) M_PI / nbLatBands;
2425         float sinTheta, cosTheta;
2426
2427         sincosf(theta, &sinTheta, &cosTheta);
2428
2429         for (unsigned lon = 0; lon <= nbLonBands; lon++) {
2430             float phi = lon * 2 * (float) M_PI / nbLonBands;
2431             float sinPhi, cosPhi;
2432
2433             sincosf(phi, &sinPhi, &cosPhi);
2434
2435             float x = cosPhi * sinTheta;
2436             float y = cosTheta;
2437             float z = sinPhi * sinTheta;
2438
2439             unsigned off1 = lat * (nbLonBands + 1) + lon;
2440             dst_data[off1].position.x = SPHERE_RADIUS * x;
2441             dst_data[off1].position.y = SPHERE_RADIUS * y;
2442             dst_data[off1].position.z = SPHERE_RADIUS * z;
2443
2444             dst_data[off1].texture.x = lon / (float) nbLonBands; // 0(left) to 1(right)
2445             dst_data[off1].texture.y = lat / (float) nbLatBands; // 0(top) to 1 (bottom)
2446         }
2447     }
2448
2449     for (unsigned lat = 0; lat < nbLatBands; lat++) {
2450         for (unsigned lon = 0; lon < nbLonBands; lon++) {
2451             unsigned first = (lat * (nbLonBands + 1)) + lon;
2452             unsigned second = first + nbLonBands + 1;
2453
2454             unsigned off = (lat * nbLatBands + lon) * 3 * 2;
2455
2456             triangle_pos[off] = first;
2457             triangle_pos[off + 1] = first + 1;
2458             triangle_pos[off + 2] = second;
2459
2460             triangle_pos[off + 3] = second;
2461             triangle_pos[off + 4] = first + 1;
2462             triangle_pos[off + 5] = second + 1;
2463         }
2464     }
2465 }
2466
2467 static bool AllocQuadVertices(vout_display_t *vd, d3d_quad_t *quad,
2468                               video_projection_mode_t projection)
2469 {
2470     HRESULT hr;
2471     vout_display_sys_t *sys = vd->sys;
2472
2473     if (projection == PROJECTION_MODE_RECTANGULAR)
2474     {
2475         quad->vertexCount = 4;
2476         quad->indexCount = 2 * 3;
2477     }
2478     else if (projection == PROJECTION_MODE_EQUIRECTANGULAR)
2479     {
2480         quad->vertexCount = (SPHERE_SLICES+1) * (SPHERE_SLICES+1);
2481         quad->indexCount = nbLatBands * nbLonBands * 2 * 3;
2482     }
2483     else
2484     {
2485         msg_Warn(vd, "Projection mode %d not handled", projection);
2486         return false;
2487     }
2488
2489     D3D11_BUFFER_DESC bd;
2490     memset(&bd, 0, sizeof(bd));
2491     bd.Usage = D3D11_USAGE_DYNAMIC;
2492     bd.ByteWidth = sizeof(d3d_vertex_t) * quad->vertexCount;
2493     bd.BindFlags = D3D11_BIND_VERTEX_BUFFER;
2494     bd.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
2495
2496     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &bd, NULL, &quad->pVertexBuffer);
2497     if(FAILED(hr)) {
2498       msg_Err(vd, "Failed to create vertex buffer. (hr=%lX)", hr);
2499       return false;
2500     }
2501
2502     /* create the index of the vertices */
2503     D3D11_BUFFER_DESC quadDesc = {
2504         .Usage = D3D11_USAGE_DYNAMIC,
2505         .ByteWidth = sizeof(WORD) * quad->indexCount,
2506         .BindFlags = D3D11_BIND_INDEX_BUFFER,
2507         .CPUAccessFlags = D3D11_CPU_ACCESS_WRITE,
2508     };
2509
2510     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &quadDesc, NULL, &quad->pIndexBuffer);
2511     if(FAILED(hr)) {
2512         msg_Err(vd, "Could not create the quad indices. (hr=0x%lX)", hr);
2513         return false;
2514     }
2515
2516     return true;
2517 }
2518
2519 static bool UpdateQuadPosition( vout_display_t *vd, d3d_quad_t *quad,
2520                                 const RECT *output,
2521                                 video_projection_mode_t projection,
2522                                 video_orientation_t orientation )
2523 {
2524     vout_display_sys_t *sys = vd->sys;
2525     HRESULT hr;
2526     D3D11_MAPPED_SUBRESOURCE mappedResource;
2527
2528     /* create the vertices */
2529     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2530     if (FAILED(hr)) {
2531         msg_Err(vd, "Failed to lock the vertex buffer (hr=0x%lX)", hr);
2532         return false;
2533     }
2534     d3d_vertex_t *dst_data = mappedResource.pData;
2535
2536     /* create the vertex indices */
2537     hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pIndexBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2538     if (FAILED(hr)) {
2539         msg_Err(vd, "Failed to lock the index buffer (hr=0x%lX)", hr);
2540         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0);
2541         return false;
2542     }
2543     WORD *triangle_pos = mappedResource.pData;
2544
2545     if ( projection == PROJECTION_MODE_RECTANGULAR )
2546         SetupQuadFlat(dst_data, output, quad, triangle_pos, orientation);
2547     else
2548         SetupQuadSphere(dst_data, triangle_pos);
2549
2550     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pIndexBuffer, 0);
2551     ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pVertexBuffer, 0);
2552
2553     return true;
2554 }
2555
2556 static int SetupQuad(vout_display_t *vd, const video_format_t *fmt, d3d_quad_t *quad,
2557                      const RECT *output,
2558                      const d3d_format_t *cfg, ID3D11PixelShader *d3dpixelShader,
2559                      video_projection_mode_t projection, video_orientation_t orientation)
2560 {
2561     vout_display_sys_t *sys = vd->sys;
2562     HRESULT hr;
2563     const bool RGB_shader = IsRGBShader(cfg);
2564
2565     /* pixel shader constant buffer */
2566     PS_CONSTANT_BUFFER defaultConstants = {
2567       .Opacity = 1,
2568     };
2569     static_assert((sizeof(PS_CONSTANT_BUFFER)%16)==0,"Constant buffers require 16-byte alignment");
2570     D3D11_BUFFER_DESC constantDesc = {
2571         .Usage = D3D11_USAGE_DYNAMIC,
2572         .ByteWidth = sizeof(PS_CONSTANT_BUFFER),
2573         .BindFlags = D3D11_BIND_CONSTANT_BUFFER,
2574         .CPUAccessFlags = D3D11_CPU_ACCESS_WRITE,
2575     };
2576     D3D11_SUBRESOURCE_DATA constantInit = { .pSysMem = &defaultConstants };
2577     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, &constantInit, &quad->pPixelShaderConstants[0]);
2578     if(FAILED(hr)) {
2579         msg_Err(vd, "Could not create the pixel shader constant buffer. (hr=0x%lX)", hr);
2580         goto error;
2581     }
2582
2583     FLOAT itu_black_level = 0.f;
2584     FLOAT itu_achromacy   = 0.f;
2585     if (!RGB_shader)
2586     {
2587         switch (cfg->bitsPerChannel)
2588         {
2589         case 8:
2590             /* Rec. ITU-R BT.709-6 §4.6 */
2591             itu_black_level  =              16.f / 255.f;
2592             itu_achromacy    =             128.f / 255.f;
2593             break;
2594         case 10:
2595             /* Rec. ITU-R BT.709-6 §4.6 */
2596             itu_black_level  =              64.f / 1023.f;
2597             itu_achromacy    =             512.f / 1023.f;
2598             break;
2599         case 12:
2600             /* Rec. ITU-R BT.2020-2 Table 5 */
2601             itu_black_level  =               256.f / 4095.f;
2602             itu_achromacy    =              2048.f / 4095.f;
2603             break;
2604         default:
2605             /* unknown bitdepth, use approximation for infinite bit depth */
2606             itu_black_level  =              16.f / 256.f;
2607             itu_achromacy    =             128.f / 256.f;
2608             break;
2609         }
2610     }
2611
2612     static const FLOAT IDENTITY_4X4[4 * 4] = {
2613         1.f, 0.f, 0.f, 0.f,
2614         0.f, 1.f, 0.f, 0.f,
2615         0.f, 0.f, 1.f, 0.f,
2616         0.f, 0.f, 0.f, 1.f,
2617     };
2618
2619     /* matrices for studio range */
2620     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion, in studio range */
2621     static const FLOAT COLORSPACE_BT601_TO_FULL[4*4] = {
2622         1.164383561643836f,                 0.f,  1.596026785714286f, 0.f,
2623         1.164383561643836f, -0.391762290094914f, -0.812967647237771f, 0.f,
2624         1.164383561643836f,  2.017232142857142f,                 0.f, 0.f,
2625                        0.f,                 0.f,                 0.f, 1.f,
2626     };
2627     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion, in studio range */
2628     static const FLOAT COLORSPACE_BT709_TO_FULL[4*4] = {
2629         1.164383561643836f,                 0.f,  1.792741071428571f, 0.f,
2630         1.164383561643836f, -0.213248614273730f, -0.532909328559444f, 0.f,
2631         1.164383561643836f,  2.112401785714286f,                 0.f, 0.f,
2632                        0.f,                 0.f,                 0.f, 1.f,
2633     };
2634     /* see https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.2020_conversion, in studio range */
2635     static const FLOAT COLORSPACE_BT2020_TO_FULL[4*4] = {
2636         1.164383561643836f,  0.000000000000f,  1.678674107143f, 0.f,
2637         1.164383561643836f, -0.127007098661f, -0.440987687946f, 0.f,
2638         1.164383561643836f,  2.141772321429f,  0.000000000000f, 0.f,
2639                        0.f,              0.f,              0.f, 1.f,
2640     };
2641
2642     PS_COLOR_TRANSFORM colorspace;
2643
2644     memcpy(colorspace.WhitePoint, IDENTITY_4X4, sizeof(colorspace.WhitePoint));
2645
2646     const FLOAT *ppColorspace;
2647     if (RGB_shader)
2648         ppColorspace = IDENTITY_4X4;
2649     else {
2650         switch (fmt->space){
2651             case COLOR_SPACE_BT709:
2652                 ppColorspace = COLORSPACE_BT709_TO_FULL;
2653                 break;
2654             case COLOR_SPACE_BT2020:
2655                 ppColorspace = COLORSPACE_BT2020_TO_FULL;
2656                 break;
2657             case COLOR_SPACE_BT601:
2658                 ppColorspace = COLORSPACE_BT601_TO_FULL;
2659                 break;
2660             default:
2661             case COLOR_SPACE_UNDEF:
2662                 if( fmt->i_height > 576 )
2663                     ppColorspace = COLORSPACE_BT709_TO_FULL;
2664                 else
2665                     ppColorspace = COLORSPACE_BT601_TO_FULL;
2666                 break;
2667         }
2668         /* all matrices work in studio range and output in full range */
2669         colorspace.WhitePoint[0*4 + 3] = -itu_black_level;
2670         colorspace.WhitePoint[1*4 + 3] = -itu_achromacy;
2671         colorspace.WhitePoint[2*4 + 3] = -itu_achromacy;
2672     }
2673
2674     memcpy(colorspace.Colorspace, ppColorspace, sizeof(colorspace.Colorspace));
2675
2676     constantInit.pSysMem = &colorspace;
2677
2678     static_assert((sizeof(PS_COLOR_TRANSFORM)%16)==0,"Constant buffers require 16-byte alignment");
2679     constantDesc.ByteWidth = sizeof(PS_COLOR_TRANSFORM);
2680     hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, &constantInit, &quad->pPixelShaderConstants[1]);
2681     if(FAILED(hr)) {
2682         msg_Err(vd, "Could not create the pixel shader constant buffer. (hr=0x%lX)", hr);
2683         goto error;
2684     }
2685     quad->PSConstantsCount = 2;
2686
2687     /* vertex shader constant buffer */
2688     if ( projection == PROJECTION_MODE_EQUIRECTANGULAR )
2689     {
2690         constantDesc.ByteWidth = sizeof(VS_PROJECTION_CONST);
2691         static_assert((sizeof(VS_PROJECTION_CONST)%16)==0,"Constant buffers require 16-byte alignment");
2692         hr = ID3D11Device_CreateBuffer(sys->d3ddevice, &constantDesc, NULL, &quad->pVertexShaderConstants);
2693         if(FAILED(hr)) {
2694             msg_Err(vd, "Could not create the vertex shader constant buffer. (hr=0x%lX)", hr);
2695             goto error;
2696         }
2697
2698         SetQuadVSProjection( vd, quad, &vd->cfg->viewpoint );
2699     }
2700
2701     quad->picSys.formatTexture = cfg->formatTexture;
2702     quad->picSys.context = sys->d3dcontext;
2703     ID3D11DeviceContext_AddRef(quad->picSys.context);
2704
2705     if (!AllocQuadVertices(vd, quad, projection))
2706         goto error;
2707     if (!UpdateQuadPosition(vd, quad, output, projection, orientation))
2708         goto error;
2709
2710     quad->d3dpixelShader = d3dpixelShader;
2711     if (projection == PROJECTION_MODE_RECTANGULAR)
2712         quad->d3dvertexShader = sys->flatVSShader;
2713     else
2714         quad->d3dvertexShader = sys->projectionVSShader;
2715
2716     return VLC_SUCCESS;
2717
2718 error:
2719     ReleaseQuad(quad);
2720     return VLC_EGENERIC;
2721 }
2722
2723 static void ReleaseQuad(d3d_quad_t *quad)
2724 {
2725     if (quad->pPixelShaderConstants[0])
2726     {
2727         ID3D11Buffer_Release(quad->pPixelShaderConstants[0]);
2728         quad->pPixelShaderConstants[0] = NULL;
2729     }
2730     if (quad->pPixelShaderConstants[1])
2731     {
2732         ID3D11Buffer_Release(quad->pPixelShaderConstants[1]);
2733         quad->pPixelShaderConstants[1] = NULL;
2734     }
2735     if (quad->pVertexBuffer)
2736     {
2737         ID3D11Buffer_Release(quad->pVertexBuffer);
2738         quad->pVertexBuffer = NULL;
2739     }
2740     quad->d3dvertexShader = NULL;
2741     if (quad->pIndexBuffer)
2742     {
2743         ID3D11Buffer_Release(quad->pIndexBuffer);
2744         quad->pIndexBuffer = NULL;
2745     }
2746     if (quad->pVertexShaderConstants)
2747     {
2748         ID3D11Buffer_Release(quad->pVertexShaderConstants);
2749         quad->pVertexShaderConstants = NULL;
2750     }
2751     ReleasePictureSys(&quad->picSys);
2752 }
2753
2754 static void Direct3D11DestroyResources(vout_display_t *vd)
2755 {
2756     vout_display_sys_t *sys = vd->sys;
2757
2758     Direct3D11DestroyPool(vd);
2759
2760     ReleaseQuad(&sys->picQuad);
2761     Direct3D11DeleteRegions(sys->d3dregion_count, sys->d3dregions);
2762     sys->d3dregion_count = 0;
2763
2764     ReleasePictureSys(&sys->stagingSys);
2765
2766     if (sys->flatVSShader)
2767     {
2768         ID3D11VertexShader_Release(sys->flatVSShader);
2769         sys->flatVSShader = NULL;
2770     }
2771     if (sys->projectionVSShader)
2772     {
2773         ID3D11VertexShader_Release(sys->projectionVSShader);
2774         sys->projectionVSShader = NULL;
2775     }
2776     if (sys->d3drenderTargetView)
2777     {
2778         ID3D11RenderTargetView_Release(sys->d3drenderTargetView);
2779         sys->d3drenderTargetView = NULL;
2780     }
2781     if (sys->d3ddepthStencilView)
2782     {
2783         ID3D11DepthStencilView_Release(sys->d3ddepthStencilView);
2784         sys->d3ddepthStencilView = NULL;
2785     }
2786     if (sys->pSPUPixelShader)
2787     {
2788         ID3D11PixelShader_Release(sys->pSPUPixelShader);
2789         sys->pSPUPixelShader = NULL;
2790     }
2791     if (sys->picQuadPixelShader)
2792     {
2793         ID3D11PixelShader_Release(sys->picQuadPixelShader);
2794         sys->picQuadPixelShader = NULL;
2795     }
2796 #if defined(HAVE_ID3D11VIDEODECODER)
2797     if( sys->context_lock != INVALID_HANDLE_VALUE )
2798     {
2799         CloseHandle( sys->context_lock );
2800         sys->context_lock = INVALID_HANDLE_VALUE;
2801     }
2802 #endif
2803
2804     msg_Dbg(vd, "Direct3D11 resources destroyed");
2805 }
2806
2807 static void Direct3D11DeleteRegions(int count, picture_t **region)
2808 {
2809     for (int i = 0; i < count; ++i) {
2810         if (region[i]) {
2811             picture_Release(region[i]);
2812         }
2813     }
2814     free(region);
2815 }
2816
2817 static void DestroyPictureQuad(picture_t *p_picture)
2818 {
2819     ReleaseQuad( (d3d_quad_t *) p_picture->p_sys );
2820     free( p_picture );
2821 }
2822
2823 static void UpdateQuadOpacity(vout_display_t *vd, const d3d_quad_t *quad, float opacity)
2824 {
2825     vout_display_sys_t *sys = vd->sys;
2826     D3D11_MAPPED_SUBRESOURCE mappedResource;
2827
2828     HRESULT hr = ID3D11DeviceContext_Map(sys->d3dcontext, (ID3D11Resource *)quad->pPixelShaderConstants[0], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2829     if (SUCCEEDED(hr)) {
2830         FLOAT *dst_data = mappedResource.pData;
2831         *dst_data = opacity;
2832         ID3D11DeviceContext_Unmap(sys->d3dcontext, (ID3D11Resource *)quad->pPixelShaderConstants[0], 0);
2833     }
2834     else {
2835         msg_Err(vd, "Failed to lock the subpicture vertex buffer (hr=0x%lX)", hr);
2836     }
2837 }
2838
2839 static int Direct3D11MapSubpicture(vout_display_t *vd, int *subpicture_region_count,
2840                                    picture_t ***region, subpicture_t *subpicture)
2841 {
2842     vout_display_sys_t *sys = vd->sys;
2843     D3D11_MAPPED_SUBRESOURCE mappedResource;
2844     D3D11_TEXTURE2D_DESC texDesc;
2845     HRESULT hr;
2846     int err;
2847
2848     if (sys->d3dregion_format == NULL)
2849         return VLC_EGENERIC;
2850
2851     int count = 0;
2852     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
2853         count++;
2854
2855     *region = calloc(count, sizeof(picture_t *));
2856     if (unlikely(*region==NULL))
2857         return VLC_ENOMEM;
2858     *subpicture_region_count = count;
2859
2860     int i = 0;
2861     for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next, i++) {
2862         if (!r->fmt.i_width || !r->fmt.i_height)
2863             continue; // won't render anything, keep the cache for later
2864
2865         for (int j = 0; j < sys->d3dregion_count; j++) {
2866             picture_t *cache = sys->d3dregions[j];
2867             if (cache != NULL && ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX]) {
2868                 ID3D11Texture2D_GetDesc( ((d3d_quad_t *) cache->p_sys)->picSys.texture[KNOWN_DXGI_INDEX], &texDesc );
2869                 if (texDesc.Format == sys->d3dregion_format->formatTexture &&
2870                     texDesc.Width  == r->fmt.i_visible_width &&
2871                     texDesc.Height == r->fmt.i_visible_height) {
2872                     (*region)[i] = cache;
2873                     memset(&sys->d3dregions[j], 0, sizeof(cache)); // do not reuse this cached value
2874                     break;
2875                 }
2876             }
2877         }
2878
2879         picture_t *quad_picture = (*region)[i];
2880         if (quad_picture == NULL) {
2881             ID3D11Texture2D *textures[D3D11_MAX_SHADER_VIEW] = {0};
2882             d3d_quad_t *d3dquad = calloc(1, sizeof(*d3dquad));
2883             if (unlikely(d3dquad==NULL)) {
2884                 continue;
2885             }
2886             if (AllocateTextures(vd, sys->d3dregion_format, &r->fmt, 1, textures, true)) {
2887                 msg_Err(vd, "Failed to allocate %dx%d texture for OSD",
2888                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2889                 for (int i=0; i<D3D11_MAX_SHADER_VIEW; i++)
2890                     if (textures[i])
2891                         ID3D11Texture2D_Release(textures[i]);
2892                 free(d3dquad);
2893                 continue;
2894             }
2895
2896             for (unsigned plane = 0; plane < D3D11_MAX_SHADER_VIEW; plane++) {
2897                 d3dquad->picSys.texture[plane] = textures[plane];
2898             }
2899             if (AllocateShaderView(VLC_OBJECT(vd), sys->d3ddevice, sys->d3dregion_format,
2900                                    d3dquad->picSys.texture, 0,
2901                                    d3dquad->picSys.resourceView)) {
2902                 msg_Err(vd, "Failed to create %dx%d shader view for OSD",
2903                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2904                 free(d3dquad);
2905                 continue;
2906             }
2907             d3dquad->i_width    = r->fmt.i_width;
2908             d3dquad->i_height   = r->fmt.i_height;
2909             RECT output;
2910             output.left   = r->fmt.i_x_offset;
2911             output.right  = r->fmt.i_x_offset + r->fmt.i_width;
2912             output.top    = r->fmt.i_y_offset;
2913             output.bottom = r->fmt.i_y_offset + r->fmt.i_height;
2914
2915             err = SetupQuad( vd, &r->fmt, d3dquad, &output,
2916                              sys->d3dregion_format, sys->pSPUPixelShader,
2917                              PROJECTION_MODE_RECTANGULAR, ORIENT_NORMAL );
2918             if (err != VLC_SUCCESS) {
2919                 msg_Err(vd, "Failed to create %dx%d quad for OSD",
2920                         r->fmt.i_visible_width, r->fmt.i_visible_height);
2921                 free(d3dquad);
2922                 continue;
2923             }
2924             picture_resource_t picres = {
2925                 .p_sys      = (picture_sys_t *) d3dquad,
2926                 .pf_destroy = DestroyPictureQuad,
2927             };
2928             (*region)[i] = picture_NewFromResource(&r->fmt, &picres);
2929             if ((*region)[i] == NULL) {
2930                 msg_Err(vd, "Failed to create %dx%d picture for OSD",
2931                         r->fmt.i_width, r->fmt.i_height);
2932                 ReleaseQuad(d3dquad);
2933                 continue;
2934             }
2935             quad_picture = (*region)[i];
2936         }
2937
2938         hr = ID3D11DeviceContext_Map(sys->d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0, D3D11_MAP_WRITE_DISCARD, 0, &mappedResource);
2939         if( SUCCEEDED(hr) ) {
2940             err = CommonUpdatePicture(quad_picture, NULL, mappedResource.pData, mappedResource.RowPitch);
2941             if (err != VLC_SUCCESS) {
2942                 msg_Err(vd, "Failed to set the buffer on the SPU picture" );
2943                 picture_Release(quad_picture);
2944                 continue;
2945             }
2946
2947             picture_CopyPixels(quad_picture, r->p_picture);
2948
2949             ID3D11DeviceContext_Unmap(sys->d3dcontext, ((d3d_quad_t *) quad_picture->p_sys)->picSys.resource[KNOWN_DXGI_INDEX], 0);
2950         } else {
2951             msg_Err(vd, "Failed to map the SPU texture (hr=0x%lX)", hr );
2952             picture_Release(quad_picture);
2953             continue;
2954         }
2955
2956         d3d_quad_t *quad = (d3d_quad_t *) quad_picture->p_sys;
2957
2958         quad->cropViewport.Width =  (FLOAT) r->fmt.i_visible_width  * RECTWidth(sys->sys.rect_dest)  / subpicture->i_original_picture_width;
2959         quad->cropViewport.Height = (FLOAT) r->fmt.i_visible_height * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
2960         quad->cropViewport.MinDepth = 0.0f;
2961         quad->cropViewport.MaxDepth = 1.0f;
2962         quad->cropViewport.TopLeftX = sys->sys.rect_dest.left + (FLOAT) r->i_x * RECTWidth(sys->sys.rect_dest) / subpicture->i_original_picture_width;
2963         quad->cropViewport.TopLeftY = sys->sys.rect_dest.top  + (FLOAT) r->i_y * RECTHeight(sys->sys.rect_dest) / subpicture->i_original_picture_height;
2964
2965         UpdateQuadOpacity(vd, quad, r->i_alpha / 255.0f );
2966     }
2967     return VLC_SUCCESS;
2968 }
2969