ScreenDisplayCaptureSourceMac.mm [plain text]
/*
* Copyright (C) 2017 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "ScreenDisplayCaptureSourceMac.h"
#if ENABLE(MEDIA_STREAM) && PLATFORM(MAC)
#include "GraphicsContextCG.h"
#include "ImageBuffer.h"
#include "Logging.h"
#include "MediaConstraints.h"
#include "MediaSampleAVFObjC.h"
#include "NotImplemented.h"
#include "PlatformLayer.h"
#include "RealtimeMediaSourceSettings.h"
#include "CoreVideoSoftLink.h"
extern "C" {
size_t CGDisplayModeGetPixelsWide(CGDisplayModeRef);
size_t CGDisplayModeGetPixelsHigh(CGDisplayModeRef);
}
namespace WebCore {
static int32_t roundUpToMacroblockMultiple(int32_t size)
{
return (size + 15) & ~15;
}
std::optional<CGDirectDisplayID> ScreenDisplayCaptureSourceMac::updateDisplayID(CGDirectDisplayID displayID)
{
uint32_t displayCount = 0;
auto err = CGGetActiveDisplayList(0, nullptr, &displayCount);
if (err) {
RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get display count", static_cast<int>(err));
return std::nullopt;
}
if (!displayCount) {
RELEASE_LOG(Media, "CGGetActiveDisplayList() returned a display count of 0");
return std::nullopt;
}
CGDirectDisplayID activeDisplays[displayCount];
err = CGGetActiveDisplayList(displayCount, &(activeDisplays[0]), &displayCount);
if (err) {
RELEASE_LOG(Media, "CGGetActiveDisplayList() returned error %d when trying to get the active display list", static_cast<int>(err));
return std::nullopt;
}
auto displayMask = CGDisplayIDToOpenGLDisplayMask(displayID);
for (auto display : activeDisplays) {
if (displayMask == CGDisplayIDToOpenGLDisplayMask(display))
return display;
}
return std::nullopt;
}
CaptureSourceOrError ScreenDisplayCaptureSourceMac::create(const String& deviceID, const MediaConstraints* constraints)
{
bool ok;
auto displayID = deviceID.toUIntStrict(&ok);
if (!ok) {
RELEASE_LOG(Media, "Display ID does not convert to 32-bit integer");
return { };
}
auto actualDisplayID = updateDisplayID(displayID);
if (!actualDisplayID)
return { };
auto source = adoptRef(*new ScreenDisplayCaptureSourceMac(actualDisplayID.value()));
if (constraints && source->applyConstraints(*constraints))
return { };
return CaptureSourceOrError(WTFMove(source));
}
ScreenDisplayCaptureSourceMac::ScreenDisplayCaptureSourceMac(uint32_t displayID)
: DisplayCaptureSourceCocoa("Screen")
, m_displayID(displayID)
{
}
ScreenDisplayCaptureSourceMac::~ScreenDisplayCaptureSourceMac()
{
if (m_observingDisplayChanges)
CGDisplayRemoveReconfigurationCallback(displayReconfigurationCallBack, this);
m_currentFrame = nullptr;
}
bool ScreenDisplayCaptureSourceMac::createDisplayStream()
{
static const int screenQueueMaximumLength = 6;
auto actualDisplayID = updateDisplayID(m_displayID);
if (!actualDisplayID) {
captureFailed();
return false;
}
if (m_displayID != actualDisplayID.value()) {
m_displayID = actualDisplayID.value();
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
}
if (!m_displayStream) {
if (size().isEmpty()) {
RetainPtr<CGDisplayModeRef> displayMode = adoptCF(CGDisplayCopyDisplayMode(m_displayID));
auto screenWidth = CGDisplayModeGetPixelsWide(displayMode.get());
auto screenHeight = CGDisplayModeGetPixelsHigh(displayMode.get());
if (!screenWidth || !screenHeight) {
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), unable to get screen width/height", this);
captureFailed();
return false;
}
setWidth(screenWidth);
setHeight(screenHeight);
}
if (!m_captureQueue)
m_captureQueue = adoptOSObject(dispatch_queue_create("ScreenDisplayCaptureSourceMac Capture Queue", DISPATCH_QUEUE_SERIAL));
double frameTime = 1 / frameRate();
auto frameTimeCF = adoptCF(CFNumberCreate(nullptr, kCFNumberDoubleType, &frameTime));
int depth = screenQueueMaximumLength;
auto depthCF = adoptCF(CFNumberCreate(nullptr, kCFNumberIntType, &depth));
CFTypeRef keys[] = {
kCGDisplayStreamMinimumFrameTime,
kCGDisplayStreamQueueDepth,
kCGDisplayStreamColorSpace,
kCGDisplayStreamShowCursor,
};
CFTypeRef values[] = {
frameTimeCF.get(),
depthCF.get(),
sRGBColorSpaceRef(),
kCFBooleanTrue,
};
auto streamOptions = adoptCF(CFDictionaryCreate(kCFAllocatorDefault, keys, values, WTF_ARRAY_LENGTH(keys), &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
auto weakThis = makeWeakPtr(*this);
m_frameAvailableBlock = Block_copy(^(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef) {
if (!weakThis)
return;
weakThis->frameAvailable(status, displayTime, frameSurface, updateRef);
});
m_displayStream = adoptCF(CGDisplayStreamCreateWithDispatchQueue(m_displayID, size().width(), size().height(), kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, streamOptions.get(), m_captureQueue.get(), m_frameAvailableBlock));
if (!m_displayStream) {
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::createDisplayStream(%p), CGDisplayStreamCreate failed", this);
captureFailed();
return false;
}
}
if (!m_observingDisplayChanges) {
CGDisplayRegisterReconfigurationCallback(displayReconfigurationCallBack, this);
m_observingDisplayChanges = true;
}
return true;
}
void ScreenDisplayCaptureSourceMac::startProducingData()
{
DisplayCaptureSourceCocoa::startProducingData();
if (m_isRunning)
return;
startDisplayStream();
}
void ScreenDisplayCaptureSourceMac::stopProducingData()
{
DisplayCaptureSourceCocoa::stopProducingData();
if (!m_isRunning)
return;
if (m_displayStream)
CGDisplayStreamStop(m_displayStream.get());
m_isRunning = false;
}
RetainPtr<CMSampleBufferRef> ScreenDisplayCaptureSourceMac::sampleBufferFromPixelBuffer(CVPixelBufferRef pixelBuffer)
{
if (!pixelBuffer)
return nullptr;
CMTime sampleTime = CMTimeMake(((elapsedTime() + 100_ms) * 100).seconds(), 100);
CMSampleTimingInfo timingInfo = { kCMTimeInvalid, sampleTime, sampleTime };
CMVideoFormatDescriptionRef formatDescription = nullptr;
auto status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, &formatDescription);
if (status) {
RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
return nullptr;
}
CMSampleBufferRef sampleBuffer;
status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, (CVImageBufferRef)pixelBuffer, formatDescription, &timingInfo, &sampleBuffer);
CFRelease(formatDescription);
if (status) {
RELEASE_LOG(Media, "Failed to initialize CMSampleBuffer with error code: %d", static_cast<int>(status));
return nullptr;
}
return adoptCF(sampleBuffer);
}
RetainPtr<CVPixelBufferRef> ScreenDisplayCaptureSourceMac::pixelBufferFromIOSurface(IOSurfaceRef surface)
{
if (!m_bufferAttributes) {
m_bufferAttributes = adoptCF(CFDictionaryCreateMutable(nullptr, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks));
auto format = IOSurfaceGetPixelFormat(surface);
if (format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange || format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
// If the width x height isn't a multiple of 16 x 16 and the surface has extra memory in the planes, set pixel buffer attributes to reflect it.
auto width = IOSurfaceGetWidth(surface);
auto height = IOSurfaceGetHeight(surface);
int32_t extendedRight = roundUpToMacroblockMultiple(width) - width;
int32_t extendedBottom = roundUpToMacroblockMultiple(height) - height;
if ((IOSurfaceGetBytesPerRowOfPlane(surface, 0) >= width + extendedRight)
&& (IOSurfaceGetBytesPerRowOfPlane(surface, 1) >= width + extendedRight)
&& (IOSurfaceGetAllocSize(surface) >= (height + extendedBottom) * IOSurfaceGetBytesPerRowOfPlane(surface, 0) * 3 / 2)) {
auto cfInt = adoptCF(CFNumberCreate(nullptr, kCFNumberIntType, &extendedRight));
CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsRightKey, cfInt.get());
cfInt = adoptCF(CFNumberCreate(nullptr, kCFNumberIntType, &extendedBottom));
CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferExtendedPixelsBottomKey, cfInt.get());
}
}
CFDictionarySetValue(m_bufferAttributes.get(), kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
}
CVPixelBufferRef pixelBuffer;
auto status = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, m_bufferAttributes.get(), &pixelBuffer);
if (status) {
RELEASE_LOG(Media, "Failed to initialize CMVideoFormatDescription with error code: %d", static_cast<int>(status));
return nullptr;
}
return adoptCF(pixelBuffer);
}
void ScreenDisplayCaptureSourceMac::generateFrame()
{
if (!m_currentFrame.ioSurface())
return;
DisplaySurface currentFrame;
{
LockHolder lock(m_currentFrameMutex);
currentFrame = m_currentFrame.ioSurface();
}
auto pixelBuffer = pixelBufferFromIOSurface(currentFrame.ioSurface());
if (!pixelBuffer)
return;
auto sampleBuffer = sampleBufferFromPixelBuffer(pixelBuffer.get());
if (!sampleBuffer)
return;
videoSampleAvailable(MediaSampleAVFObjC::create(sampleBuffer.get()));
}
void ScreenDisplayCaptureSourceMac::startDisplayStream()
{
auto actualDisplayID = updateDisplayID(m_displayID);
if (!actualDisplayID)
return;
if (m_displayID != actualDisplayID.value()) {
m_displayID = actualDisplayID.value();
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::create(%p), display ID changed to %d", this, static_cast<int>(m_displayID));
}
if (!m_displayStream && !createDisplayStream())
return;
auto err = CGDisplayStreamStart(m_displayStream.get());
if (err) {
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::startProducingData(%p), CGDisplayStreamStart failed with error %d", this, static_cast<int>(err));
captureFailed();
return;
}
m_isRunning = true;
}
bool ScreenDisplayCaptureSourceMac::applySize(const IntSize& newSize)
{
if (size() == newSize)
return true;
m_bufferAttributes = nullptr;
m_displayStream = nullptr;
return true;
}
bool ScreenDisplayCaptureSourceMac::applyFrameRate(double rate)
{
if (frameRate() != rate) {
m_bufferAttributes = nullptr;
m_displayStream = nullptr;
}
return DisplayCaptureSourceCocoa::applyFrameRate(rate);
}
void ScreenDisplayCaptureSourceMac::commitConfiguration()
{
if (m_isRunning && !m_displayStream)
startDisplayStream();
}
void ScreenDisplayCaptureSourceMac::displayWasReconfigured(CGDirectDisplayID, CGDisplayChangeSummaryFlags)
{
// FIXME: implement!
}
void ScreenDisplayCaptureSourceMac::displayReconfigurationCallBack(CGDirectDisplayID display, CGDisplayChangeSummaryFlags flags, void *userInfo)
{
if (userInfo)
reinterpret_cast<ScreenDisplayCaptureSourceMac *>(userInfo)->displayWasReconfigured(display, flags);
}
void ScreenDisplayCaptureSourceMac::frameAvailable(CGDisplayStreamFrameStatus status, uint64_t displayTime, IOSurfaceRef frameSurface, CGDisplayStreamUpdateRef updateRef)
{
switch (status) {
case kCGDisplayStreamFrameStatusFrameComplete:
break;
case kCGDisplayStreamFrameStatusFrameIdle:
break;
case kCGDisplayStreamFrameStatusFrameBlank:
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusFrameBlank", this);
break;
case kCGDisplayStreamFrameStatusStopped:
RELEASE_LOG(Media, "ScreenDisplayCaptureSourceMac::frameAvailable(%p), kCGDisplayStreamFrameStatusStopped", this);
break;
}
if (!frameSurface || !displayTime)
return;
size_t count;
auto* rects = CGDisplayStreamUpdateGetRects(updateRef, kCGDisplayStreamUpdateDirtyRects, &count);
if (!rects || !count)
return;
LockHolder lock(m_currentFrameMutex);
m_lastFrameTime = MonotonicTime::now();
m_currentFrame = frameSurface;
}
} // namespace WebCore
#endif // ENABLE(MEDIA_STREAM) && PLATFORM(MAC)