From 2aad81259ef832172d65aeaba6401fee857a0f48 Mon Sep 17 00:00:00 2001 From: Austin Orion Date: Thu, 7 Jan 2021 14:34:31 -0800 Subject: [PATCH] Refactor and implement WgcCapturerWin, a source agnostic capturer. This change refactors WgcWindowCapturer into WgcCapturerWin, a source agnostic capturer, and finishes the implementation to enable both window and screen capture. This CL depends on another which must complete first: 196622: Add ability to load CreateDirect3DDeviceFromDXGIDevice from d3d11.dll | https://webrtc-review.googlesource.com/c/src/+/196622 This feature remains off by default behind a build flag, due to it adding a depency on the Win10 SDK vv10.0.19041 which not all consumers of WebRTC have upgraded to. A follow up change later will enable the rtc_enable_win_wgc build flag, but for now it should remain off. The basic operation of this class is as follows: Consumers call either WgcCapturerWin::CreateRawWindowCapturer or CreateRawScreenCapturer to receive a correctly initialized WgcCapturerWin object suitable for the desired source type. Callers then indicate via SelectSource and a SourceId the desired capture target, and the capturer creates an appropriate WgcCaptureSource for the correct type (window or screen) using the WgcCaptureSourceFactory supplied at construction. Next, callers request frames for the currently selected source and the capturer then creates a WgcCaptureSession and stores it in a map for more efficient capture of multiple sources. The WgcCaptureSession is supplied with a GraphicsCaptureItem created by the WgcCaptureSource. It uses this item to create a Direct3D11CaptureFramePool and create and start a GraphicsCaptureSession. Once started, captured frames will begin to be deposited into the FramePool. Typically, one would listen for the FrameArrived event and process the frame then, but due to the synchronous nature of the DesktopCapturer interface, and to avoid a more complicated multi- threaded architecture we ignore the FrameArrived event. Instead, we wait for a request for a frame from the caller, then we check the FramePool for a frame, and process it on demand. Processing a frame involves moving the image data from an ID3D11Texture2D stored in the GPU into a texture that is accessible from the CPU, and then copying the data into the new WgcDesktopFrame class. This copy is necessary as otherwise we would need to manage the lifetimes of the CaptureFrame and ID3D11Texture2D objects, lest the buffer be invalidated. Once we've copied the data and returned it to the caller, we can unmap the texture and exit the scope of the GetFrame method, which will destruct the CaptureFrame object. At this point, the CaptureSession will begin capturing a new frame, and will soon deposit it into the FramePool and we can repeat. Bug: webrtc:9273 Change-Id: I02263c4fd587df652b04d5267fad8965330d0f5b Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/200161 Reviewed-by: Jamie Walch Reviewed-by: Guido Urdaneta Commit-Queue: Austin Orion Cr-Commit-Position: refs/heads/master@{#33083} --- modules/desktop_capture/BUILD.gn | 11 +- modules/desktop_capture/desktop_capturer.cc | 14 +- .../win/screen_capture_utils.cc | 41 ++ .../win/screen_capture_utils.h | 9 + .../win/screen_capture_utils_unittest.cc | 20 + .../win/wgc_capture_session.cc | 244 +++++++++++- .../desktop_capture/win/wgc_capture_session.h | 77 +++- .../desktop_capture/win/wgc_capture_source.cc | 127 ++++++ .../desktop_capture/win/wgc_capture_source.h | 122 ++++++ ...apturer_win_wgc.cc => wgc_capturer_win.cc} | 98 +++-- .../desktop_capture/win/wgc_capturer_win.h | 131 +++++++ .../win/wgc_capturer_win_unittest.cc | 361 ++++++++++++++++++ .../desktop_capture/win/wgc_desktop_frame.cc | 25 ++ .../desktop_capture/win/wgc_desktop_frame.h | 46 +++ .../win/window_capturer_win_wgc.h | 69 ---- 15 files changed, 1273 insertions(+), 122 deletions(-) create mode 100644 modules/desktop_capture/win/wgc_capture_source.cc create mode 100644 modules/desktop_capture/win/wgc_capture_source.h rename modules/desktop_capture/win/{window_capturer_win_wgc.cc => wgc_capturer_win.cc} (51%) create mode 100644 modules/desktop_capture/win/wgc_capturer_win.h create mode 100644 modules/desktop_capture/win/wgc_capturer_win_unittest.cc create mode 100644 modules/desktop_capture/win/wgc_desktop_frame.cc create mode 100644 modules/desktop_capture/win/wgc_desktop_frame.h delete mode 100644 modules/desktop_capture/win/window_capturer_win_wgc.h diff --git a/modules/desktop_capture/BUILD.gn b/modules/desktop_capture/BUILD.gn index 70344e5ba8..e8b4ba8b08 100644 --- a/modules/desktop_capture/BUILD.gn +++ b/modules/desktop_capture/BUILD.gn @@ -143,6 +143,9 @@ if (rtc_include_tests) { if (is_mac) { sources += [ "screen_capturer_mac_unittest.cc" ] } + if (rtc_enable_win_wgc) { + sources += [ "win/wgc_capturer_win_unittest.cc" ] + } deps += [ ":desktop_capture_mock" ] public_configs = [ ":x11_config" ] } @@ -564,8 +567,12 @@ rtc_library("desktop_capture_generic") { sources += [ "win/wgc_capture_session.cc", "win/wgc_capture_session.h", - "win/window_capturer_win_wgc.cc", - "win/window_capturer_win_wgc.h", + "win/wgc_capture_source.cc", + "win/wgc_capture_source.h", + "win/wgc_capturer_win.cc", + "win/wgc_capturer_win.h", + "win/wgc_desktop_frame.cc", + "win/wgc_desktop_frame.h", ] defines += [ "RTC_ENABLE_WIN_WGC" ] diff --git a/modules/desktop_capture/desktop_capturer.cc b/modules/desktop_capture/desktop_capturer.cc index e1fff4ea57..e90fce4dda 100644 --- a/modules/desktop_capture/desktop_capturer.cc +++ b/modules/desktop_capture/desktop_capturer.cc @@ -21,7 +21,7 @@ #include "modules/desktop_capture/desktop_capturer_differ_wrapper.h" #if defined(RTC_ENABLE_WIN_WGC) -#include "modules/desktop_capture/win/window_capturer_win_wgc.h" +#include "modules/desktop_capture/win/wgc_capturer_win.h" #include "rtc_base/win/windows_version.h" const bool kUseWinWgcCapturer = false; @@ -61,7 +61,7 @@ std::unique_ptr DesktopCapturer::CreateWindowCapturer( // fully implemented. if (kUseWinWgcCapturer && rtc::rtc_win::GetVersion() >= rtc::rtc_win::Version::VERSION_WIN10_RS5) { - return WindowCapturerWinWgc::CreateRawWindowCapturer(options); + return WgcCapturerWin::CreateRawWindowCapturer(options); } #endif // defined(RTC_ENABLE_WIN_WGC) @@ -82,6 +82,16 @@ std::unique_ptr DesktopCapturer::CreateWindowCapturer( // static std::unique_ptr DesktopCapturer::CreateScreenCapturer( const DesktopCaptureOptions& options) { +#if defined(RTC_ENABLE_WIN_WGC) + // TODO(bugs.webrtc.org/11760): Add a WebRTC field trial (or similar + // mechanism) check here that leads to use of the WGC capturer once it is + // fully implemented. + if (kUseWinWgcCapturer && + rtc::rtc_win::GetVersion() >= rtc::rtc_win::Version::VERSION_WIN10_RS5) { + return WgcCapturerWin::CreateRawScreenCapturer(options); + } +#endif // defined(RTC_ENABLE_WIN_WGC) + std::unique_ptr capturer = CreateRawScreenCapturer(options); if (capturer && options.detect_updated_region()) { capturer.reset(new DesktopCapturerDifferWrapper(std::move(capturer))); diff --git a/modules/desktop_capture/win/screen_capture_utils.cc b/modules/desktop_capture/win/screen_capture_utils.cc index 95f6d92059..caa1716bc2 100644 --- a/modules/desktop_capture/win/screen_capture_utils.cc +++ b/modules/desktop_capture/win/screen_capture_utils.cc @@ -21,6 +21,41 @@ #include "rtc_base/win32.h" namespace webrtc { +namespace { + +BOOL CALLBACK GetMonitorListHandler(HMONITOR monitor, + HDC hdc, + LPRECT rect, + LPARAM data) { + auto monitor_list = reinterpret_cast(data); + + // Get the name of the monitor. + MONITORINFOEXA monitor_info; + monitor_info.cbSize = sizeof(MONITORINFOEXA); + if (!GetMonitorInfoA(monitor, &monitor_info)) + return FALSE; + + DesktopCapturer::Source monitor_source; + monitor_source.id = reinterpret_cast(monitor); + monitor_source.title = monitor_info.szDevice; + monitor_list->push_back(monitor_source); + return TRUE; +} + +} // namespace + +// |monitors| is populated with HMONITOR values for each display monitor found. +// This is in contrast to |GetScreenList| which returns the display indices. +bool GetMonitorList(DesktopCapturer::SourceList* monitors) { + RTC_DCHECK_EQ(monitors->size(), 0U); + // |EnumDisplayMonitors| accepts a display context and a rectangle, which + // allows us to specify a certain region and return only the monitors that + // intersect that region. We, however, want all the monitors, so we pass in + // NULL parameters. + return EnumDisplayMonitors(/*hdc=*/NULL, /*clip_rect=*/NULL, + GetMonitorListHandler, + reinterpret_cast(monitors)); +} bool GetScreenList(DesktopCapturer::SourceList* screens, std::vector* device_names /* = nullptr */) { @@ -51,6 +86,12 @@ bool GetScreenList(DesktopCapturer::SourceList* screens, return true; } +bool IsMonitorValid(DesktopCapturer::SourceId monitor) { + MONITORINFO monitor_info; + monitor_info.cbSize = sizeof(MONITORINFO); + return GetMonitorInfoA(reinterpret_cast(monitor), &monitor_info); +} + bool IsScreenValid(DesktopCapturer::SourceId screen, std::wstring* device_key) { if (screen == kFullDesktopScreenId) { *device_key = L""; diff --git a/modules/desktop_capture/win/screen_capture_utils.h b/modules/desktop_capture/win/screen_capture_utils.h index 5c4c11d542..1edd744c93 100644 --- a/modules/desktop_capture/win/screen_capture_utils.h +++ b/modules/desktop_capture/win/screen_capture_utils.h @@ -19,6 +19,10 @@ namespace webrtc { +// Output the HMONITOR values of all display monitors into |monitors|. Returns +// true if succeeded, or false if it fails to enumerate the display monitors. +bool GetMonitorList(DesktopCapturer::SourceList* monitors); + // Output the list of active screens into |screens|. Returns true if succeeded, // or false if it fails to enumerate the display devices. If the |device_names| // is provided, it will be filled with the DISPLAY_DEVICE.DeviceName in UTF-8 @@ -27,6 +31,11 @@ namespace webrtc { bool GetScreenList(DesktopCapturer::SourceList* screens, std::vector* device_names = nullptr); +// Returns true if |monitor| is an HMONITOR that represents a valid display +// monitor. Consumers should check that the results of |GetMonitorList| are +// valid before use if a WM_DISPLAYCHANGE message has been received. +bool IsMonitorValid(DesktopCapturer::SourceId monitor); + // Returns true if |screen| is a valid screen. The screen device key is // returned through |device_key| if the screen is valid. The device key can be // used in GetScreenRect to verify the screen matches the previously obtained diff --git a/modules/desktop_capture/win/screen_capture_utils_unittest.cc b/modules/desktop_capture/win/screen_capture_utils_unittest.cc index a71c4f7610..1ced2a0a04 100644 --- a/modules/desktop_capture/win/screen_capture_utils_unittest.cc +++ b/modules/desktop_capture/win/screen_capture_utils_unittest.cc @@ -29,4 +29,24 @@ TEST(ScreenCaptureUtilsTest, GetScreenList) { ASSERT_EQ(screens.size(), device_names.size()); } +TEST(ScreenCaptureUtilsTest, GetMonitorList) { + DesktopCapturer::SourceList monitors; + + ASSERT_TRUE(GetMonitorList(&monitors)); + ASSERT_GT(monitors.size(), 0ULL); +} + +TEST(ScreenCaptureUtilsTest, IsMonitorValid) { + DesktopCapturer::SourceList monitors; + + ASSERT_TRUE(GetMonitorList(&monitors)); + ASSERT_GT(monitors.size(), 0ULL); + + ASSERT_TRUE(IsMonitorValid(monitors[0].id)); +} + +TEST(ScreenCaptureUtilsTest, InvalidMonitor) { + ASSERT_FALSE(IsMonitorValid(NULL)); +} + } // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capture_session.cc b/modules/desktop_capture/win/wgc_capture_session.cc index ee55cf6164..f8ba6d403c 100644 --- a/modules/desktop_capture/win/wgc_capture_session.cc +++ b/modules/desktop_capture/win/wgc_capture_session.cc @@ -10,31 +10,261 @@ #include "modules/desktop_capture/win/wgc_capture_session.h" +#include +#include +#include +#include #include +#include +#include "modules/desktop_capture/win/wgc_desktop_frame.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/win/create_direct3d_device.h" +#include "rtc_base/win/get_activation_factory.h" using Microsoft::WRL::ComPtr; +namespace WGC = ABI::Windows::Graphics::Capture; + namespace webrtc { +namespace { + +// We must use a BGRA pixel format that has 4 bytes per pixel, as required by +// the DesktopFrame interface. +const auto kPixelFormat = ABI::Windows::Graphics::DirectX::DirectXPixelFormat:: + DirectXPixelFormat_B8G8R8A8UIntNormalized; + +// We only want 1 buffer in our frame pool to reduce latency. If we had more, +// they would sit in the pool for longer and be stale by the time we are asked +// for a new frame. +const int kNumBuffers = 1; + +} // namespace WgcCaptureSession::WgcCaptureSession(ComPtr d3d11_device, - HWND window) - : d3d11_device_(std::move(d3d11_device)), window_(window) {} + ComPtr item) + : d3d11_device_(std::move(d3d11_device)), item_(std::move(item)) {} WgcCaptureSession::~WgcCaptureSession() = default; HRESULT WgcCaptureSession::StartCapture() { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK(!is_capture_started_); - RTC_DCHECK(d3d11_device_); - RTC_DCHECK(window_); - return E_NOTIMPL; + if (item_closed_) { + RTC_LOG(LS_ERROR) << "The target source has been closed."; + return E_ABORT; + } + + RTC_DCHECK(d3d11_device_); + RTC_DCHECK(item_); + + // Listen for the Closed event, to detect if the source we are capturing is + // closed (e.g. application window is closed or monitor is disconnected). If + // it is, we should abort the capture. + auto closed_handler = + Microsoft::WRL::Callback>( + this, &WgcCaptureSession::OnItemClosed); + EventRegistrationToken item_closed_token; + HRESULT hr = item_->add_Closed(closed_handler.Get(), &item_closed_token); + if (FAILED(hr)) + return hr; + + ComPtr dxgi_device; + hr = d3d11_device_->QueryInterface(IID_PPV_ARGS(&dxgi_device)); + if (FAILED(hr)) + return hr; + + if (!ResolveCoreWinRTDirect3DDelayload()) + return E_FAIL; + + hr = CreateDirect3DDeviceFromDXGIDevice(dxgi_device.Get(), &direct3d_device_); + if (FAILED(hr)) + return hr; + + ComPtr frame_pool_statics; + hr = GetActivationFactory< + ABI::Windows::Graphics::Capture::IDirect3D11CaptureFramePoolStatics, + RuntimeClass_Windows_Graphics_Capture_Direct3D11CaptureFramePool>( + &frame_pool_statics); + if (FAILED(hr)) + return hr; + + // Cast to FramePoolStatics2 so we can use CreateFreeThreaded and avoid the + // need to have a DispatcherQueue. We don't listen for the FrameArrived event, + // so there's no difference. + ComPtr frame_pool_statics2; + hr = frame_pool_statics->QueryInterface(IID_PPV_ARGS(&frame_pool_statics2)); + if (FAILED(hr)) + return hr; + + ABI::Windows::Graphics::SizeInt32 item_size; + hr = item_.Get()->get_Size(&item_size); + if (FAILED(hr)) + return hr; + + previous_size_ = item_size; + + hr = frame_pool_statics2->CreateFreeThreaded(direct3d_device_.Get(), + kPixelFormat, kNumBuffers, + item_size, &frame_pool_); + if (FAILED(hr)) + return hr; + + hr = frame_pool_->CreateCaptureSession(item_.Get(), &session_); + if (FAILED(hr)) + return hr; + + hr = session_->StartCapture(); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "Failed to start CaptureSession: " << hr; + return hr; + } + + is_capture_started_ = true; + return hr; } -HRESULT WgcCaptureSession::GetMostRecentFrame( +HRESULT WgcCaptureSession::GetFrame( std::unique_ptr* output_frame) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + if (item_closed_) { + RTC_LOG(LS_ERROR) << "The target source has been closed."; + return E_ABORT; + } + RTC_DCHECK(is_capture_started_); - return E_NOTIMPL; + ComPtr capture_frame; + HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "TryGetNextFrame failed: " << hr; + return hr; + } + + if (!capture_frame) + return hr; + + // We need to get this CaptureFrame as an ID3D11Texture2D so that we can get + // the raw image data in the format required by the DesktopFrame interface. + ComPtr + d3d_surface; + hr = capture_frame->get_Surface(&d3d_surface); + if (FAILED(hr)) + return hr; + + ComPtr + direct3DDxgiInterfaceAccess; + hr = d3d_surface->QueryInterface(IID_PPV_ARGS(&direct3DDxgiInterfaceAccess)); + if (FAILED(hr)) + return hr; + + ComPtr texture_2D; + hr = direct3DDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&texture_2D)); + if (FAILED(hr)) + return hr; + + if (!mapped_texture_) { + hr = CreateMappedTexture(texture_2D); + if (FAILED(hr)) + return hr; + } + + // We need to copy |texture_2D| into |mapped_texture_| as the latter has the + // D3D11_CPU_ACCESS_READ flag set, which lets us access the image data. + // Otherwise it would only be readable by the GPU. + ComPtr d3d_context; + d3d11_device_->GetImmediateContext(&d3d_context); + d3d_context->CopyResource(mapped_texture_.Get(), texture_2D.Get()); + + D3D11_MAPPED_SUBRESOURCE map_info; + hr = d3d_context->Map(mapped_texture_.Get(), /*subresource_index=*/0, + D3D11_MAP_READ, /*D3D11_MAP_FLAG_DO_NOT_WAIT=*/0, + &map_info); + if (FAILED(hr)) + return hr; + + ABI::Windows::Graphics::SizeInt32 new_size; + hr = capture_frame->get_ContentSize(&new_size); + if (FAILED(hr)) + return hr; + + // If the size has changed since the last capture, we must be sure to choose + // the smaller of the two sizes. Otherwise we might overrun our buffer, or + // read stale data from the last frame. + int previous_area = previous_size_.Width * previous_size_.Height; + int new_area = new_size.Width * new_size.Height; + auto smaller_size = previous_area < new_area ? previous_size_ : new_size; + + // Make a copy of the data pointed to by |map_info.pData| so we are free to + // unmap our texture. + uint8_t* data = static_cast(map_info.pData); + int data_size = smaller_size.Height * map_info.RowPitch; + std::vector image_data(data, data + data_size); + DesktopSize size(smaller_size.Width, smaller_size.Height); + + // Transfer ownership of |image_data| to the output_frame. + *output_frame = std::make_unique( + size, static_cast(map_info.RowPitch), std::move(image_data)); + + d3d_context->Unmap(mapped_texture_.Get(), 0); + + // If the size changed, we must resize the texture and frame pool to fit the + // new size. + if (previous_area != new_area) { + hr = CreateMappedTexture(texture_2D, new_size.Width, new_size.Height); + if (FAILED(hr)) + return hr; + + hr = frame_pool_->Recreate(direct3d_device_.Get(), kPixelFormat, + kNumBuffers, new_size); + if (FAILED(hr)) + return hr; + } + + previous_size_ = new_size; + return hr; +} + +HRESULT WgcCaptureSession::CreateMappedTexture( + ComPtr src_texture, + UINT width, + UINT height) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + D3D11_TEXTURE2D_DESC src_desc; + src_texture->GetDesc(&src_desc); + D3D11_TEXTURE2D_DESC map_desc; + map_desc.Width = width == 0 ? src_desc.Width : width; + map_desc.Height = height == 0 ? src_desc.Height : height; + map_desc.MipLevels = src_desc.MipLevels; + map_desc.ArraySize = src_desc.ArraySize; + map_desc.Format = src_desc.Format; + map_desc.SampleDesc = src_desc.SampleDesc; + map_desc.Usage = D3D11_USAGE_STAGING; + map_desc.BindFlags = 0; + map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + map_desc.MiscFlags = 0; + return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_); +} + +HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender, + IInspectable* event_args) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + + RTC_LOG(LS_INFO) << "Capture target has been closed."; + item_closed_ = true; + is_capture_started_ = false; + + mapped_texture_ = nullptr; + session_ = nullptr; + frame_pool_ = nullptr; + direct3d_device_ = nullptr; + item_ = nullptr; + d3d11_device_ = nullptr; + + return S_OK; } } // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capture_session.h b/modules/desktop_capture/win/wgc_capture_session.h index 9f41331c92..0a118a1642 100644 --- a/modules/desktop_capture/win/wgc_capture_session.h +++ b/modules/desktop_capture/win/wgc_capture_session.h @@ -11,36 +11,97 @@ #ifndef MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURE_SESSION_H_ #define MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURE_SESSION_H_ -#include #include +#include #include #include -#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/win/wgc_capture_source.h" +#include "rtc_base/synchronization/sequence_checker.h" namespace webrtc { class WgcCaptureSession final { public: - WgcCaptureSession(Microsoft::WRL::ComPtr d3d11_device, - HWND window); + WgcCaptureSession( + Microsoft::WRL::ComPtr d3d11_device, + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem> item); - // Disallow copy and assign + // Disallow copy and assign. WgcCaptureSession(const WgcCaptureSession&) = delete; WgcCaptureSession& operator=(const WgcCaptureSession&) = delete; ~WgcCaptureSession(); HRESULT StartCapture(); - HRESULT GetMostRecentFrame(std::unique_ptr* output_frame); - bool IsCaptureStarted() const { return is_capture_started_; } + + // Returns a frame from the frame pool, if any are present. + HRESULT GetFrame(std::unique_ptr* output_frame); + + bool IsCaptureStarted() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); + return is_capture_started_; + } private: + // Initializes |mapped_texture_| with the properties of the |src_texture|, + // overrides the values of some necessary properties like the + // D3D11_CPU_ACCESS_READ flag. Also has optional parameters for what size + // |mapped_texture_| should be, if they aren't provided we will use the size + // of |src_texture|. + HRESULT CreateMappedTexture( + Microsoft::WRL::ComPtr src_texture, + UINT width = 0, + UINT height = 0); + + // Event handler for |item_|'s Closed event. + HRESULT OnItemClosed( + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem* sender, + IInspectable* event_args); + // A Direct3D11 Device provided by the caller. We use this to create an // IDirect3DDevice, and also to create textures that will hold the image data. Microsoft::WRL::ComPtr d3d11_device_; - HWND window_; + + // This item represents what we are capturing, we use it to create the + // capture session, and also to listen for the Closed event. + Microsoft::WRL::ComPtr + item_; + + // The IDirect3DDevice is necessary to instantiate the frame pool. + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::DirectX::Direct3D11::IDirect3DDevice> + direct3d_device_; + + // The frame pool is where frames are deposited during capture, we retrieve + // them from here with TryGetNextFrame(). + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IDirect3D11CaptureFramePool> + frame_pool_; + + // This texture holds the final image data. We made it a member so we can + // reuse it, instead of having to create a new texture every time we grab a + // frame. + Microsoft::WRL::ComPtr mapped_texture_; + + // This lets us know when the source has been resized, which is important + // because we must resize the framepool and our texture to be able to hold + // enough data for the frame. + ABI::Windows::Graphics::SizeInt32 previous_size_; + + // The capture session lets us set properties about the capture before it + // starts such as whether to capture the mouse cursor, and it lets us tell WGC + // to start capturing frames. + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureSession> + session_; + + bool item_closed_ = false; bool is_capture_started_ = false; + + SequenceChecker sequence_checker_; }; } // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capture_source.cc b/modules/desktop_capture/win/wgc_capture_source.cc new file mode 100644 index 0000000000..b7eb62f201 --- /dev/null +++ b/modules/desktop_capture/win/wgc_capture_source.cc @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/win/wgc_capture_source.h" + +#include +#include + +#include "modules/desktop_capture/win/screen_capture_utils.h" +#include "modules/desktop_capture/win/window_capture_utils.h" +#include "rtc_base/win/get_activation_factory.h" + +using Microsoft::WRL::ComPtr; +namespace WGC = ABI::Windows::Graphics::Capture; + +namespace webrtc { + +WgcCaptureSource::WgcCaptureSource(DesktopCapturer::SourceId source_id) + : source_id_(source_id) {} +WgcCaptureSource::~WgcCaptureSource() = default; + +HRESULT WgcCaptureSource::GetCaptureItem( + ComPtr* result) { + HRESULT hr = S_OK; + if (!item_) + hr = CreateCaptureItem(&item_); + + *result = item_; + return hr; +} + +WgcCaptureSourceFactory::~WgcCaptureSourceFactory() = default; + +WgcWindowSourceFactory::WgcWindowSourceFactory() = default; +WgcWindowSourceFactory::~WgcWindowSourceFactory() = default; + +std::unique_ptr WgcWindowSourceFactory::CreateCaptureSource( + DesktopCapturer::SourceId source_id) { + return std::make_unique(source_id); +} + +WgcScreenSourceFactory::WgcScreenSourceFactory() = default; +WgcScreenSourceFactory::~WgcScreenSourceFactory() = default; + +std::unique_ptr WgcScreenSourceFactory::CreateCaptureSource( + DesktopCapturer::SourceId source_id) { + return std::make_unique(source_id); +} + +WgcWindowSource::WgcWindowSource(DesktopCapturer::SourceId source_id) + : WgcCaptureSource(source_id) {} +WgcWindowSource::~WgcWindowSource() = default; + +bool WgcWindowSource::IsCapturable() { + return IsWindowValidAndVisible(reinterpret_cast(GetSourceId())); +} + +HRESULT WgcWindowSource::CreateCaptureItem( + ComPtr* result) { + if (!ResolveCoreWinRTDelayload()) + return E_FAIL; + + ComPtr interop; + HRESULT hr = GetActivationFactory< + IGraphicsCaptureItemInterop, + RuntimeClass_Windows_Graphics_Capture_GraphicsCaptureItem>(&interop); + if (FAILED(hr)) + return hr; + + ComPtr item; + hr = interop->CreateForWindow(reinterpret_cast(GetSourceId()), + IID_PPV_ARGS(&item)); + if (FAILED(hr)) + return hr; + + if (!item) + return E_HANDLE; + + *result = std::move(item); + return hr; +} + +WgcScreenSource::WgcScreenSource(DesktopCapturer::SourceId source_id) + : WgcCaptureSource(source_id) {} +WgcScreenSource::~WgcScreenSource() = default; + +bool WgcScreenSource::IsCapturable() { + // 0 is the id used to capture all display monitors, so it is valid. + if (GetSourceId() == 0) + return true; + + return IsMonitorValid(GetSourceId()); +} + +HRESULT WgcScreenSource::CreateCaptureItem( + ComPtr* result) { + if (!ResolveCoreWinRTDelayload()) + return E_FAIL; + + ComPtr interop; + HRESULT hr = GetActivationFactory< + IGraphicsCaptureItemInterop, + RuntimeClass_Windows_Graphics_Capture_GraphicsCaptureItem>(&interop); + if (FAILED(hr)) + return hr; + + ComPtr item; + hr = interop->CreateForMonitor(reinterpret_cast(GetSourceId()), + IID_PPV_ARGS(&item)); + if (FAILED(hr)) + return hr; + + if (!item) + return E_HANDLE; + + *result = std::move(item); + return hr; +} + +} // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capture_source.h b/modules/desktop_capture/win/wgc_capture_source.h new file mode 100644 index 0000000000..20ccdfb853 --- /dev/null +++ b/modules/desktop_capture/win/wgc_capture_source.h @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURE_SOURCE_H_ +#define MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURE_SOURCE_H_ + +#include +#include +#include + +#include "modules/desktop_capture/desktop_capturer.h" +namespace webrtc { + +// Abstract class to represent the source that WGC-based capturers capture +// from. Could represent an application window or a screen. Consumers should use +// the appropriate Wgc*SourceFactory class to create WgcCaptureSource objects +// of the appropriate type. +class WgcCaptureSource { + public: + explicit WgcCaptureSource(DesktopCapturer::SourceId source_id); + virtual ~WgcCaptureSource(); + + virtual bool IsCapturable() = 0; + HRESULT GetCaptureItem( + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>* result); + DesktopCapturer::SourceId GetSourceId() { return source_id_; } + + protected: + virtual HRESULT CreateCaptureItem( + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>* result) = 0; + + private: + Microsoft::WRL::ComPtr + item_; + DesktopCapturer::SourceId source_id_; +}; + +class WgcCaptureSourceFactory { + public: + virtual ~WgcCaptureSourceFactory(); + + virtual std::unique_ptr CreateCaptureSource( + DesktopCapturer::SourceId) = 0; +}; + +class WgcWindowSourceFactory final : public WgcCaptureSourceFactory { + public: + WgcWindowSourceFactory(); + + // Disallow copy and assign. + WgcWindowSourceFactory(const WgcWindowSourceFactory&) = delete; + WgcWindowSourceFactory& operator=(const WgcWindowSourceFactory&) = delete; + + ~WgcWindowSourceFactory() override; + + std::unique_ptr CreateCaptureSource( + DesktopCapturer::SourceId) override; +}; + +class WgcScreenSourceFactory final : public WgcCaptureSourceFactory { + public: + WgcScreenSourceFactory(); + + WgcScreenSourceFactory(const WgcScreenSourceFactory&) = delete; + WgcScreenSourceFactory& operator=(const WgcScreenSourceFactory&) = delete; + + ~WgcScreenSourceFactory() override; + + std::unique_ptr CreateCaptureSource( + DesktopCapturer::SourceId) override; +}; + +// Class for capturing application windows. +class WgcWindowSource final : public WgcCaptureSource { + public: + explicit WgcWindowSource(DesktopCapturer::SourceId source_id); + + WgcWindowSource(const WgcWindowSource&) = delete; + WgcWindowSource& operator=(const WgcWindowSource&) = delete; + + ~WgcWindowSource() override; + + bool IsCapturable() override; + + private: + HRESULT CreateCaptureItem( + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>* result) + override; +}; + +// Class for capturing screens/monitors/displays. +class WgcScreenSource final : public WgcCaptureSource { + public: + explicit WgcScreenSource(DesktopCapturer::SourceId source_id); + + WgcScreenSource(const WgcScreenSource&) = delete; + WgcScreenSource& operator=(const WgcScreenSource&) = delete; + + ~WgcScreenSource() override; + + bool IsCapturable() override; + + private: + HRESULT CreateCaptureItem( + Microsoft::WRL::ComPtr< + ABI::Windows::Graphics::Capture::IGraphicsCaptureItem>* result) + override; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURE_SOURCE_H_ diff --git a/modules/desktop_capture/win/window_capturer_win_wgc.cc b/modules/desktop_capture/win/wgc_capturer_win.cc similarity index 51% rename from modules/desktop_capture/win/window_capturer_win_wgc.cc rename to modules/desktop_capture/win/wgc_capturer_win.cc index 30a672d9ef..4c5ca29dc3 100644 --- a/modules/desktop_capture/win/window_capturer_win_wgc.cc +++ b/modules/desktop_capture/win/wgc_capturer_win.cc @@ -8,31 +8,51 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/desktop_capture/win/window_capturer_win_wgc.h" +#include "modules/desktop_capture/win/wgc_capturer_win.h" #include +#include "modules/desktop_capture/win/wgc_desktop_frame.h" #include "rtc_base/logging.h" +namespace WGC = ABI::Windows::Graphics::Capture; +using Microsoft::WRL::ComPtr; + namespace webrtc { -WindowCapturerWinWgc::WindowCapturerWinWgc() = default; -WindowCapturerWinWgc::~WindowCapturerWinWgc() = default; +WgcCapturerWin::WgcCapturerWin( + std::unique_ptr source_factory, + std::unique_ptr source_enumerator) + : source_factory_(std::move(source_factory)), + source_enumerator_(std::move(source_enumerator)) {} +WgcCapturerWin::~WgcCapturerWin() = default; -bool WindowCapturerWinWgc::GetSourceList(SourceList* sources) { - return window_capture_helper_.EnumerateCapturableWindows(sources); +// static +std::unique_ptr WgcCapturerWin::CreateRawWindowCapturer( + const DesktopCaptureOptions& options) { + return std::make_unique( + std::make_unique(), + std::make_unique()); } -bool WindowCapturerWinWgc::SelectSource(SourceId id) { - HWND window = reinterpret_cast(id); - if (!IsWindowValidAndVisible(window)) - return false; - - window_ = window; - return true; +// static +std::unique_ptr WgcCapturerWin::CreateRawScreenCapturer( + const DesktopCaptureOptions& options) { + return std::make_unique( + std::make_unique(), + std::make_unique()); } -void WindowCapturerWinWgc::Start(Callback* callback) { +bool WgcCapturerWin::GetSourceList(SourceList* sources) { + return source_enumerator_->FindAllSources(sources); +} + +bool WgcCapturerWin::SelectSource(DesktopCapturer::SourceId id) { + capture_source_ = source_factory_->CreateCaptureSource(id); + return capture_source_->IsCapturable(); +} + +void WgcCapturerWin::Start(Callback* callback) { RTC_DCHECK(!callback_); RTC_DCHECK(callback); @@ -62,11 +82,11 @@ void WindowCapturerWinWgc::Start(Callback* callback) { } } -void WindowCapturerWinWgc::CaptureFrame() { +void WgcCapturerWin::CaptureFrame() { RTC_DCHECK(callback_); - if (!window_) { - RTC_LOG(LS_ERROR) << "Window hasn't been selected"; + if (!capture_source_) { + RTC_LOG(LS_ERROR) << "Source hasn't been selected"; callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, /*frame=*/nullptr); return; @@ -79,29 +99,36 @@ void WindowCapturerWinWgc::CaptureFrame() { return; } + HRESULT hr; WgcCaptureSession* capture_session = nullptr; - auto iter = ongoing_captures_.find(window_); - if (iter == ongoing_captures_.end()) { - auto iter_success_pair = ongoing_captures_.emplace( - std::piecewise_construct, std::forward_as_tuple(window_), - std::forward_as_tuple(d3d11_device_, window_)); - if (iter_success_pair.second) { - capture_session = &iter_success_pair.first->second; - } else { - RTC_LOG(LS_ERROR) << "Failed to create new WgcCaptureSession."; + std::map::iterator session_iter = + ongoing_captures_.find(capture_source_->GetSourceId()); + if (session_iter == ongoing_captures_.end()) { + ComPtr item; + hr = capture_source_->GetCaptureItem(&item); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "Failed to create a GraphicsCaptureItem: " << hr; callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, /*frame=*/nullptr); return; } + + std::pair::iterator, bool> + iter_success_pair = ongoing_captures_.emplace( + std::piecewise_construct, + std::forward_as_tuple(capture_source_->GetSourceId()), + std::forward_as_tuple(d3d11_device_, item)); + RTC_DCHECK(iter_success_pair.second); + capture_session = &iter_success_pair.first->second; } else { - capture_session = &iter->second; + capture_session = &session_iter->second; } - HRESULT hr; if (!capture_session->IsCaptureStarted()) { hr = capture_session->StartCapture(); if (FAILED(hr)) { RTC_LOG(LS_ERROR) << "Failed to start capture: " << hr; + ongoing_captures_.erase(capture_source_->GetSourceId()); callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, /*frame=*/nullptr); return; @@ -109,16 +136,16 @@ void WindowCapturerWinWgc::CaptureFrame() { } std::unique_ptr frame; - hr = capture_session->GetMostRecentFrame(&frame); + hr = capture_session->GetFrame(&frame); if (FAILED(hr)) { - RTC_LOG(LS_ERROR) << "GetMostRecentFrame failed: " << hr; + RTC_LOG(LS_ERROR) << "GetFrame failed: " << hr; + ongoing_captures_.erase(capture_source_->GetSourceId()); callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_PERMANENT, /*frame=*/nullptr); return; } if (!frame) { - RTC_LOG(LS_WARNING) << "GetMostRecentFrame returned an empty frame."; callback_->OnCaptureResult(DesktopCapturer::Result::ERROR_TEMPORARY, /*frame=*/nullptr); return; @@ -128,10 +155,13 @@ void WindowCapturerWinWgc::CaptureFrame() { std::move(frame)); } -// static -std::unique_ptr WindowCapturerWinWgc::CreateRawWindowCapturer( - const DesktopCaptureOptions& options) { - return std::unique_ptr(new WindowCapturerWinWgc()); +bool WgcCapturerWin::IsSourceBeingCaptured(DesktopCapturer::SourceId id) { + std::map::iterator + session_iter = ongoing_captures_.find(id); + if (session_iter == ongoing_captures_.end()) + return false; + + return session_iter->second.IsCaptureStarted(); } } // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_capturer_win.h b/modules/desktop_capture/win/wgc_capturer_win.h new file mode 100644 index 0000000000..f48d6019be --- /dev/null +++ b/modules/desktop_capture/win/wgc_capturer_win.h @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURER_WIN_H_ +#define MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURER_WIN_H_ + +#include +#include +#include +#include + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/win/screen_capture_utils.h" +#include "modules/desktop_capture/win/wgc_capture_session.h" +#include "modules/desktop_capture/win/wgc_capture_source.h" +#include "modules/desktop_capture/win/window_capture_utils.h" + +namespace webrtc { + +// WgcCapturerWin is initialized with an implementation of this base class, +// which it uses to find capturable sources of a particular type. This way, +// WgcCapturerWin can remain source-agnostic. +class SourceEnumerator { + public: + virtual ~SourceEnumerator() = default; + + virtual bool FindAllSources(DesktopCapturer::SourceList* sources) = 0; +}; + +class WindowEnumerator final : public SourceEnumerator { + public: + WindowEnumerator() = default; + + WindowEnumerator(const WindowEnumerator&) = delete; + WindowEnumerator& operator=(const WindowEnumerator&) = delete; + + ~WindowEnumerator() override = default; + + bool FindAllSources(DesktopCapturer::SourceList* sources) override { + return window_capture_helper_.EnumerateCapturableWindows(sources); + } + + private: + WindowCaptureHelperWin window_capture_helper_; +}; + +class ScreenEnumerator final : public SourceEnumerator { + public: + ScreenEnumerator() = default; + + ScreenEnumerator(const ScreenEnumerator&) = delete; + ScreenEnumerator& operator=(const ScreenEnumerator&) = delete; + + ~ScreenEnumerator() override = default; + + bool FindAllSources(DesktopCapturer::SourceList* sources) override { + return webrtc::GetMonitorList(sources); + } +}; + +// A capturer that uses the Window.Graphics.Capture APIs. It is suitable for +// both window and screen capture (but only one type per instance). Consumers +// should not instantiate this class directly, instead they should use +// |CreateRawWindowCapturer()| or |CreateRawScreenCapturer()| to receive a +// capturer appropriate for the type of source they want to capture. +class WgcCapturerWin : public DesktopCapturer { + public: + WgcCapturerWin(std::unique_ptr source_factory, + std::unique_ptr source_enumerator); + + WgcCapturerWin(const WgcCapturerWin&) = delete; + WgcCapturerWin& operator=(const WgcCapturerWin&) = delete; + + ~WgcCapturerWin() override; + + static std::unique_ptr CreateRawWindowCapturer( + const DesktopCaptureOptions& options); + + static std::unique_ptr CreateRawScreenCapturer( + const DesktopCaptureOptions& options); + + // DesktopCapturer interface. + bool GetSourceList(SourceList* sources) override; + bool SelectSource(SourceId id) override; + void Start(Callback* callback) override; + void CaptureFrame() override; + + // Used in WgcCapturerTests. + bool IsSourceBeingCaptured(SourceId id); + + private: + // Factory to create a WgcCaptureSource for us whenever SelectSource is + // called. Initialized at construction with a source-specific implementation. + std::unique_ptr source_factory_; + + // The source enumerator helps us find capturable sources of the appropriate + // type. Initialized at construction with a source-specific implementation. + std::unique_ptr source_enumerator_; + + // The WgcCaptureSource represents the source we are capturing. It tells us + // if the source is capturable and it creates the GraphicsCaptureItem for us. + std::unique_ptr capture_source_; + + // A map of all the sources we are capturing and the associated + // WgcCaptureSession. Frames for the current source (indicated via + // SelectSource) will be retrieved from the appropriate session when + // requested via CaptureFrame. + // This helps us efficiently capture multiple sources (e.g. when consumers + // are trying to display a list of available capture targets with thumbnails). + std::map ongoing_captures_; + + // The callback that we deliver frames to, synchronously, before CaptureFrame + // returns. + Callback* callback_ = nullptr; + + // A Direct3D11 device that is shared amongst the WgcCaptureSessions, who + // require one to perform the capture. + Microsoft::WRL::ComPtr<::ID3D11Device> d3d11_device_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_WIN_WGC_CAPTURER_WIN_H_ diff --git a/modules/desktop_capture/win/wgc_capturer_win_unittest.cc b/modules/desktop_capture/win/wgc_capturer_win_unittest.cc new file mode 100644 index 0000000000..01af0442bb --- /dev/null +++ b/modules/desktop_capture/win/wgc_capturer_win_unittest.cc @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/win/wgc_capturer_win.h" + +#include +#include +#include + +#include "modules/desktop_capture/desktop_capture_options.h" +#include "modules/desktop_capture/desktop_capture_types.h" +#include "modules/desktop_capture/desktop_capturer.h" +#include "modules/desktop_capture/win/test_support/test_window.h" +#include "modules/desktop_capture/win/window_capture_utils.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/thread.h" +#include "rtc_base/win/scoped_com_initializer.h" +#include "rtc_base/win/windows_version.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +const char kWindowThreadName[] = "wgc_capturer_test_window_thread"; +const WCHAR kWindowTitle[] = L"WGC Capturer Test Window"; + +const int kSmallWindowWidth = 200; +const int kSmallWindowHeight = 100; +const int kWindowWidth = 300; +const int kWindowHeight = 200; +const int kLargeWindowWidth = 400; +const int kLargeWindowHeight = 300; + +// The size of the image we capture is slightly smaller than the actual size of +// the window. +const int kWindowWidthSubtrahend = 14; +const int kWindowHeightSubtrahend = 7; + +// Custom message constants so we can direct our thread to close windows +// and quit running. +const UINT kNoOp = WM_APP; +const UINT kDestroyWindow = WM_APP + 1; +const UINT kQuitRunning = WM_APP + 2; + +enum CaptureType { kWindowCapture = 0, kScreenCapture = 1 }; + +} // namespace + +class WgcCapturerWinTest : public ::testing::TestWithParam, + public DesktopCapturer::Callback { + public: + void SetUp() override { + if (rtc::rtc_win::GetVersion() < rtc::rtc_win::Version::VERSION_WIN10_RS5) { + RTC_LOG(LS_INFO) + << "Skipping WgcWindowCaptureTests on Windows versions < RS5."; + GTEST_SKIP(); + } + + com_initializer_ = + std::make_unique(ScopedCOMInitializer::kMTA); + EXPECT_TRUE(com_initializer_->Succeeded()); + } + + void SetUpForWindowCapture() { + capturer_ = WgcCapturerWin::CreateRawWindowCapturer( + DesktopCaptureOptions::CreateDefault()); + CreateWindowOnSeparateThread(); + StartWindowThreadMessageLoop(); + source_id_ = GetTestWindowIdFromSourceList(); + } + + void SetUpForScreenCapture() { + capturer_ = WgcCapturerWin::CreateRawScreenCapturer( + DesktopCaptureOptions::CreateDefault()); + source_id_ = GetScreenIdFromSourceList(); + } + + void TearDown() override { + if (window_open_) { + CloseTestWindow(); + } + } + + // The window must live on a separate thread so that we can run a message pump + // without blocking the test thread. This is necessary if we are interested in + // having GraphicsCaptureItem events (i.e. the Closed event) fire, and it more + // closely resembles how capture works in the wild. + void CreateWindowOnSeparateThread() { + window_thread_ = rtc::Thread::Create(); + window_thread_->SetName(kWindowThreadName, nullptr); + window_thread_->Start(); + window_thread_->Invoke(RTC_FROM_HERE, [this]() { + window_thread_id_ = GetCurrentThreadId(); + window_info_ = + CreateTestWindow(kWindowTitle, kWindowHeight, kWindowWidth); + window_open_ = true; + + while (!IsWindowResponding(window_info_.hwnd)) { + RTC_LOG(LS_INFO) << "Waiting for test window to become responsive in " + "WgcWindowCaptureTest."; + } + + while (!IsWindowValidAndVisible(window_info_.hwnd)) { + RTC_LOG(LS_INFO) << "Waiting for test window to be visible in " + "WgcWindowCaptureTest."; + } + }); + + ASSERT_TRUE(window_thread_->RunningForTest()); + ASSERT_FALSE(window_thread_->IsCurrent()); + } + + void StartWindowThreadMessageLoop() { + window_thread_->PostTask(RTC_FROM_HERE, [this]() { + MSG msg; + BOOL gm; + while ((gm = ::GetMessage(&msg, NULL, 0, 0)) != 0 && gm != -1) { + ::DispatchMessage(&msg); + if (msg.message == kDestroyWindow) { + DestroyTestWindow(window_info_); + } + if (msg.message == kQuitRunning) { + PostQuitMessage(0); + } + } + }); + } + + void CloseTestWindow() { + ::PostThreadMessage(window_thread_id_, kDestroyWindow, 0, 0); + ::PostThreadMessage(window_thread_id_, kQuitRunning, 0, 0); + window_thread_->Stop(); + window_open_ = false; + } + + DesktopCapturer::SourceId GetTestWindowIdFromSourceList() { + // Frequently, the test window will not show up in GetSourceList because it + // was created too recently. Since we are confident the window will be found + // eventually we loop here until we find it. + intptr_t src_id; + do { + DesktopCapturer::SourceList sources; + EXPECT_TRUE(capturer_->GetSourceList(&sources)); + + auto it = std::find_if( + sources.begin(), sources.end(), + [&](const DesktopCapturer::Source& src) { + return src.id == reinterpret_cast(window_info_.hwnd); + }); + + src_id = it->id; + } while (src_id != reinterpret_cast(window_info_.hwnd)); + + return src_id; + } + + DesktopCapturer::SourceId GetScreenIdFromSourceList() { + DesktopCapturer::SourceList sources; + EXPECT_TRUE(capturer_->GetSourceList(&sources)); + EXPECT_GT(sources.size(), 0ULL); + return sources[0].id; + } + + void DoCapture() { + // Sometimes the first few frames are empty becaues the capture engine is + // still starting up. We also may drop a few frames when the window is + // resized or un-minimized. + do { + capturer_->CaptureFrame(); + } while (result_ == DesktopCapturer::Result::ERROR_TEMPORARY); + + EXPECT_EQ(result_, DesktopCapturer::Result::SUCCESS); + EXPECT_TRUE(frame_); + } + + // DesktopCapturer::Callback interface + // The capturer synchronously invokes this method before |CaptureFrame()| + // returns. + void OnCaptureResult(DesktopCapturer::Result result, + std::unique_ptr frame) override { + result_ = result; + frame_ = std::move(frame); + } + + protected: + std::unique_ptr com_initializer_; + DWORD window_thread_id_; + std::unique_ptr window_thread_; + WindowInfo window_info_; + intptr_t source_id_; + bool window_open_ = false; + DesktopCapturer::Result result_; + std::unique_ptr frame_; + std::unique_ptr capturer_; +}; + +TEST_P(WgcCapturerWinTest, SelectValidSource) { + if (GetParam() == CaptureType::kWindowCapture) { + SetUpForWindowCapture(); + } else { + SetUpForScreenCapture(); + } + + EXPECT_TRUE(capturer_->SelectSource(source_id_)); +} + +TEST_P(WgcCapturerWinTest, SelectInvalidSource) { + if (GetParam() == CaptureType::kWindowCapture) { + capturer_ = WgcCapturerWin::CreateRawWindowCapturer( + DesktopCaptureOptions::CreateDefault()); + source_id_ = kNullWindowId; + } else { + capturer_ = WgcCapturerWin::CreateRawScreenCapturer( + DesktopCaptureOptions::CreateDefault()); + source_id_ = kInvalidScreenId; + } + + EXPECT_FALSE(capturer_->SelectSource(source_id_)); +} + +TEST_P(WgcCapturerWinTest, Capture) { + if (GetParam() == CaptureType::kWindowCapture) { + SetUpForWindowCapture(); + } else { + SetUpForScreenCapture(); + } + + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + + capturer_->Start(this); + DoCapture(); + EXPECT_GT(frame_->size().width(), 0); + EXPECT_GT(frame_->size().height(), 0); +} + +INSTANTIATE_TEST_SUITE_P(SourceAgnostic, + WgcCapturerWinTest, + ::testing::Values(CaptureType::kWindowCapture, + CaptureType::kScreenCapture)); + +// Monitor specific tests. +TEST_F(WgcCapturerWinTest, CaptureAllMonitors) { + SetUpForScreenCapture(); + // 0 (or a NULL HMONITOR) leads to WGC capturing all displays. + EXPECT_TRUE(capturer_->SelectSource(0)); + + capturer_->Start(this); + DoCapture(); + EXPECT_GT(frame_->size().width(), 0); + EXPECT_GT(frame_->size().height(), 0); +} + +// Window specific tests. +TEST_F(WgcCapturerWinTest, SelectMinimizedWindow) { + SetUpForWindowCapture(); + MinimizeTestWindow(reinterpret_cast(source_id_)); + EXPECT_FALSE(capturer_->SelectSource(source_id_)); + + UnminimizeTestWindow(reinterpret_cast(source_id_)); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); +} + +TEST_F(WgcCapturerWinTest, SelectClosedWindow) { + SetUpForWindowCapture(); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + + CloseTestWindow(); + EXPECT_FALSE(capturer_->SelectSource(source_id_)); +} + +TEST_F(WgcCapturerWinTest, ResizeWindowMidCapture) { + SetUpForWindowCapture(); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + + capturer_->Start(this); + DoCapture(); + EXPECT_EQ(frame_->size().width(), kWindowWidth - kWindowWidthSubtrahend); + EXPECT_EQ(frame_->size().height(), kWindowHeight - kWindowHeightSubtrahend); + + ResizeTestWindow(window_info_.hwnd, kLargeWindowWidth, kLargeWindowHeight); + DoCapture(); + // We don't expect to see the new size until the next capture, as the frame + // pool hadn't had a chance to resize yet. + DoCapture(); + EXPECT_EQ(frame_->size().width(), kLargeWindowWidth - kWindowWidthSubtrahend); + EXPECT_EQ(frame_->size().height(), + kLargeWindowHeight - kWindowHeightSubtrahend); + + ResizeTestWindow(window_info_.hwnd, kSmallWindowWidth, kSmallWindowHeight); + DoCapture(); + DoCapture(); + EXPECT_EQ(frame_->size().width(), kSmallWindowWidth - kWindowWidthSubtrahend); + EXPECT_EQ(frame_->size().height(), + kSmallWindowHeight - kWindowHeightSubtrahend); +} + +TEST_F(WgcCapturerWinTest, MinimizeWindowMidCapture) { + SetUpForWindowCapture(); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + + capturer_->Start(this); + + // Minmize the window and capture should continue but return temporary errors. + MinimizeTestWindow(window_info_.hwnd); + for (int i = 0; i < 10; ++i) { + capturer_->CaptureFrame(); + EXPECT_EQ(result_, DesktopCapturer::Result::ERROR_TEMPORARY); + } + + // Reopen the window and the capture should continue normally. + UnminimizeTestWindow(window_info_.hwnd); + DoCapture(); + // We can't verify the window size here because the test window does not + // repaint itself after it is unminimized, but capturing successfully is still + // a good test. +} + +TEST_F(WgcCapturerWinTest, CloseWindowMidCapture) { + SetUpForWindowCapture(); + EXPECT_TRUE(capturer_->SelectSource(source_id_)); + + capturer_->Start(this); + DoCapture(); + EXPECT_EQ(frame_->size().width(), kWindowWidth - kWindowWidthSubtrahend); + EXPECT_EQ(frame_->size().height(), kWindowHeight - kWindowHeightSubtrahend); + + CloseTestWindow(); + + // We need to call GetMessage to trigger the Closed event and the capturer's + // event handler for it. If we are too early and the Closed event hasn't + // arrived yet we should keep trying until the capturer receives it and stops. + auto* wgc_capturer = static_cast(capturer_.get()); + while (wgc_capturer->IsSourceBeingCaptured(source_id_)) { + // Since the capturer handles the Closed message, there will be no message + // for us and GetMessage will hang, unless we send ourselves a message + // first. + ::PostThreadMessage(GetCurrentThreadId(), kNoOp, 0, 0); + MSG msg; + ::GetMessage(&msg, NULL, 0, 0); + ::DispatchMessage(&msg); + } + + // Occasionally, one last frame will have made it into the frame pool before + // the window closed. The first call will consume it, and in that case we need + // to make one more call to CaptureFrame. + capturer_->CaptureFrame(); + if (result_ == DesktopCapturer::Result::SUCCESS) + capturer_->CaptureFrame(); + + EXPECT_EQ(result_, DesktopCapturer::Result::ERROR_PERMANENT); +} + +} // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_desktop_frame.cc b/modules/desktop_capture/win/wgc_desktop_frame.cc new file mode 100644 index 0000000000..dd9009120b --- /dev/null +++ b/modules/desktop_capture/win/wgc_desktop_frame.cc @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/desktop_capture/win/wgc_desktop_frame.h" + +#include + +namespace webrtc { + +WgcDesktopFrame::WgcDesktopFrame(DesktopSize size, + int stride, + std::vector&& image_data) + : DesktopFrame(size, stride, image_data.data(), nullptr), + image_data_(std::move(image_data)) {} + +WgcDesktopFrame::~WgcDesktopFrame() = default; + +} // namespace webrtc diff --git a/modules/desktop_capture/win/wgc_desktop_frame.h b/modules/desktop_capture/win/wgc_desktop_frame.h new file mode 100644 index 0000000000..0eca763f9e --- /dev/null +++ b/modules/desktop_capture/win/wgc_desktop_frame.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_DESKTOP_CAPTURE_WIN_WGC_DESKTOP_FRAME_H_ +#define MODULES_DESKTOP_CAPTURE_WIN_WGC_DESKTOP_FRAME_H_ + +#include +#include + +#include +#include + +#include "modules/desktop_capture/desktop_frame.h" +#include "modules/desktop_capture/desktop_geometry.h" + +namespace webrtc { + +// DesktopFrame implementation used by capturers that use the +// Windows.Graphics.Capture API. +class WgcDesktopFrame final : public DesktopFrame { + public: + // WgcDesktopFrame receives an rvalue reference to the |image_data| vector + // so that it can take ownership of it (and avoid a copy). + WgcDesktopFrame(DesktopSize size, + int stride, + std::vector&& image_data); + + WgcDesktopFrame(const WgcDesktopFrame&) = delete; + WgcDesktopFrame& operator=(const WgcDesktopFrame&) = delete; + + ~WgcDesktopFrame() override; + + private: + std::vector image_data_; +}; + +} // namespace webrtc + +#endif // MODULES_DESKTOP_CAPTURE_WIN_WGC_DESKTOP_FRAME_H_ diff --git a/modules/desktop_capture/win/window_capturer_win_wgc.h b/modules/desktop_capture/win/window_capturer_win_wgc.h deleted file mode 100644 index 7e05b0e541..0000000000 --- a/modules/desktop_capture/win/window_capturer_win_wgc.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#ifndef MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURER_WIN_WGC_H_ -#define MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURER_WIN_WGC_H_ - -#include -#include -#include -#include - -#include "modules/desktop_capture/desktop_capture_options.h" -#include "modules/desktop_capture/desktop_capturer.h" -#include "modules/desktop_capture/win/wgc_capture_session.h" -#include "modules/desktop_capture/win/window_capture_utils.h" - -namespace webrtc { - -class WindowCapturerWinWgc final : public DesktopCapturer { - public: - WindowCapturerWinWgc(); - - WindowCapturerWinWgc(const WindowCapturerWinWgc&) = delete; - WindowCapturerWinWgc& operator=(const WindowCapturerWinWgc&) = delete; - - ~WindowCapturerWinWgc() override; - - static std::unique_ptr CreateRawWindowCapturer( - const DesktopCaptureOptions& options); - - // DesktopCapturer interface. - void Start(Callback* callback) override; - void CaptureFrame() override; - bool GetSourceList(SourceList* sources) override; - bool SelectSource(SourceId id) override; - - private: - // The callback that we deliver frames to, synchronously, before CaptureFrame - // returns. - Callback* callback_ = nullptr; - - // HWND for the currently selected window or nullptr if a window is not - // selected. We may be capturing many other windows, but this is the window - // that we will return a frame for when CaptureFrame is called. - HWND window_ = nullptr; - - // This helps us enumerate the list of windows that we can capture. - WindowCaptureHelperWin window_capture_helper_; - - // A Direct3D11 device that is shared amongst the WgcCaptureSessions, who - // require one to perform the capture. - Microsoft::WRL::ComPtr<::ID3D11Device> d3d11_device_; - - // A map of all the windows we are capturing and the associated - // WgcCaptureSession. This is where we will get the frames for the window - // from, when requested. - std::map ongoing_captures_; -}; - -} // namespace webrtc - -#endif // MODULES_DESKTOP_CAPTURE_WIN_WINDOW_CAPTURER_WIN_WGC_H_