RealtimeVideoSource.cpp   [plain text]

/*
 * Copyright (C) 2018 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "config.h"
#include "RealtimeVideoSource.h"

#if ENABLE(MEDIA_STREAM)
#include "CaptureDevice.h"
#include "Logging.h"
#include "RealtimeMediaSourceCenter.h"
#include "RealtimeMediaSourceSettings.h"
#include "RemoteVideoSample.h"

#if PLATFORM(COCOA)
#include "ImageTransferSessionVT.h"
#endif

namespace WebCore {

RealtimeVideoSource::RealtimeVideoSource(String&& name, String&& id, String&& hashSalt)
    : RealtimeMediaSource(Type::Video, WTFMove(name), WTFMove(id), WTFMove(hashSalt))
{
}

RealtimeVideoSource::~RealtimeVideoSource()
{
#if PLATFORM(IOS_FAMILY)
    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().unsetActiveSource(*this);
#endif
}

void RealtimeVideoSource::prepareToProduceData()
{
    ASSERT(frameRate());

#if PLATFORM(IOS_FAMILY)
    RealtimeMediaSourceCenter::singleton().videoCaptureFactory().setActiveSource(*this);
#endif

    if (size().isEmpty() && !m_defaultSize.isEmpty())
        setSize(m_defaultSize);
}

const Vector<Ref<VideoPreset>>& RealtimeVideoSource::presets()
{
    if (m_presets.isEmpty())
        generatePresets();

    ASSERT(!m_presets.isEmpty());
    return m_presets;
}

void RealtimeVideoSource::setSupportedPresets(Vector<VideoPresetData>&& presetData)
{
    Vector<Ref<VideoPreset>> presets;

    for (auto& data : presetData)
        presets.append(VideoPreset::create(WTFMove(data)));

    setSupportedPresets(WTFMove(presets));
}

void RealtimeVideoSource::setSupportedPresets(const Vector<Ref<VideoPreset>>& presets)
{
    m_presets = WTF::map(presets, [](auto& preset) {
        return preset.copyRef();
    });

    for (auto& preset : m_presets) {
        std::sort(preset->frameRateRanges.begin(), preset->frameRateRanges.end(),
            [&] (const auto& a, const auto& b) -> bool {
                return a.minimum < b.minimum;
        });
    }
}

const Vector<IntSize>& RealtimeVideoSource::standardVideoSizes()
{
    static const auto sizes = makeNeverDestroyed([] {
        static IntSize videoSizes[] = {
            { 112, 112 },
            { 160, 160 },
            { 160, 120 }, // 4:3, QQVGA
            { 176, 144 }, // 4:3, QCIF
            { 192, 192 },
            { 192, 112 }, // 16:9
            { 192, 144 }, // 3:4
            { 240, 240 },
            { 240, 160 }, // 3:2, HQVGA
            { 320, 320 },
            { 320, 180 }, // 16:9
            { 320, 240 }, // 4:3, QVGA
            { 352, 288 }, // CIF
            { 480, 272 }, // 16:9
            { 480, 360 }, // 4:3
            { 480, 480 },
            { 640, 640 },
            { 640, 360 }, // 16:9, 360p nHD
            { 640, 480 }, // 4:3
            { 720, 720 },
            { 800, 600 }, // 4:3, SVGA
            { 960, 540 }, // 16:9, qHD
            { 1024, 600 }, // 16:9, WSVGA
            { 1024, 768 }, // 4:3, XGA
            { 1280, 960 }, // 4:3
            { 1280, 1024 }, // 5:4, SXGA
            { 1280, 720 }, // 16:9, WXGA
            { 1366, 768 }, // 16:9, HD
            { 1600, 1200}, // 4:3, UXGA
            { 1920, 1080 }, // 16:9, 1080p FHD
            { 2560, 1440 }, // 16:9, QHD
            { 2592, 1936 },
            { 3264, 2448 }, // 3:4
            { 3840, 2160 }, // 16:9, 4K UHD
        };
        Vector<IntSize> sizes;
        for (auto& size : videoSizes)
            sizes.append(size);

        return sizes;
    }());

    return sizes.get();
}
template <typename ValueType>
static void updateMinMax(ValueType& min, ValueType& max, ValueType value)
{
    min = std::min<ValueType>(min, value);
    max = std::max<ValueType>(max, value);
}

void RealtimeVideoSource::updateCapabilities(RealtimeMediaSourceCapabilities& capabilities)
{
    ASSERT(!presets().isEmpty());

    int minimumWidth = std::numeric_limits<int>::max();
    int maximumWidth = 0;
    int minimumHeight = std::numeric_limits<int>::max();
    int maximumHeight = 0;
    double minimumAspectRatio = std::numeric_limits<double>::max();
    double maximumAspectRatio = 0;
    double minimumFrameRate = std::numeric_limits<double>::max();
    double maximumFrameRate = 0;
    for (const auto& preset : presets()) {
        const auto& size = preset->size;
        updateMinMax(minimumWidth, maximumWidth, size.width());
        updateMinMax(minimumHeight, maximumHeight, size.height());
        updateMinMax(minimumAspectRatio, maximumAspectRatio, static_cast<double>(size.width()) / size.height());

        for (const auto& rate : preset->frameRateRanges) {
            updateMinMax(minimumFrameRate, maximumFrameRate, rate.minimum);
            updateMinMax(minimumFrameRate, maximumFrameRate, rate.maximum);
        }
    }

    if (canResizeVideoFrames()) {
        for (auto& size : standardVideoSizes()) {
            if (size.width() < minimumWidth || size.height() < minimumHeight) {
                minimumWidth = std::min(minimumWidth, size.width());
                minimumHeight = std::min(minimumHeight, size.height());
                minimumAspectRatio = std::min(minimumAspectRatio, static_cast<double>(size.width()) / size.height());
            }
        }
    }

    capabilities.setWidth({ minimumWidth, maximumWidth });
    capabilities.setHeight({ minimumHeight, maximumHeight });
    capabilities.setAspectRatio({ minimumAspectRatio, maximumAspectRatio });
    capabilities.setFrameRate({ minimumFrameRate, maximumFrameRate });
}

bool RealtimeVideoSource::supportsSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
{
    if (!width && !height && !frameRate)
        return true;

    return !!bestSupportedSizeAndFrameRate(width, height, frameRate);
}

bool RealtimeVideoSource::frameRateRangeIncludesRate(const FrameRateRange& range, double frameRate)
{
    const double epsilon = 0.001;
    return frameRate + epsilon >= range.minimum && frameRate - epsilon <= range.maximum;
}

bool RealtimeVideoSource::presetSupportsFrameRate(RefPtr<VideoPreset> preset, double frameRate)
{
    for (const auto& range : preset->frameRateRanges) {
        if (frameRateRangeIncludesRate(range, frameRate))
            return true;
    }

    return false;
}

bool RealtimeVideoSource::supportsCaptureSize(Optional<int> width, Optional<int> height, const Function<bool(const IntSize&)>&& function)
{
    if (width && height)
        return function({ width.value(), height.value() });

    if (width) {
        for (auto& size : standardVideoSizes()) {
            if (width.value() == size.width() && function({ size.width(), size.height() }))
                return true;
        }

        return false;
    }

    for (auto& size : standardVideoSizes()) {
        if (height.value() == size.height() && function({ size.width(), size.height() }))
            return true;
    }

    return false;
}

bool RealtimeVideoSource::shouldUsePreset(VideoPreset& current, VideoPreset& candidate)
{
    return candidate.size.width() <= current.size.width() && candidate.size.height() <= current.size.height() && prefersPreset(candidate);
}

Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> RealtimeVideoSource::bestSupportedSizeAndFrameRate(Optional<int> requestedWidth, Optional<int> requestedHeight, Optional<double> requestedFrameRate)
{
    if (!requestedWidth && !requestedHeight && !requestedFrameRate)
        return { };

    if (!requestedWidth && !requestedHeight && !size().isEmpty()) {
        requestedWidth = size().width();
        requestedHeight = size().height();
    }
    if (!requestedFrameRate)
        requestedFrameRate = frameRate();

    CaptureSizeAndFrameRate result;
    RefPtr<VideoPreset> exactSizePreset;
    RefPtr<VideoPreset> aspectRatioPreset;
    IntSize aspectRatioMatchSize;
    RefPtr<VideoPreset> resizePreset;
    IntSize resizeSize;

    for (const auto& preset : presets()) {
        const auto& presetSize = preset->size;

        if (!presetSupportsFrameRate(&preset.get(), requestedFrameRate.value()))
            continue;

        if (!requestedWidth && !requestedHeight) {
            result.requestedFrameRate = requestedFrameRate.value();
            return result;
        }

        // Don't look at presets smaller than the requested resolution because we never want to resize larger.
        if ((requestedWidth && presetSize.width() < requestedWidth.value()) || (requestedHeight && presetSize.height() < requestedHeight.value()))
            continue;

        auto lookForExactSizeMatch = [&] (const IntSize& size) -> bool {
            return preset->size == size;
        };
        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForExactSizeMatch))) {
            if (!exactSizePreset || prefersPreset(preset))
                exactSizePreset = &preset.get();
            continue;
        }

        IntSize encodingSize;
        auto lookForAspectRatioMatch = [this, &preset, &encodingSize] (const IntSize& size) -> bool {
            auto aspectRatio = [] (const IntSize size) -> double {
                return size.width() / static_cast<double>(size.height());
            };
            if (std::abs(aspectRatio(preset->size) - aspectRatio(size)) > 10e-7 || !canResizeVideoFrames())
                return false;

            encodingSize = size;
            return true;
        };
        if (supportsCaptureSize(requestedWidth, requestedHeight, WTFMove(lookForAspectRatioMatch))) {
            if (!aspectRatioPreset || shouldUsePreset(*aspectRatioPreset, preset)) {
                aspectRatioPreset = &preset.get();
                aspectRatioMatchSize = encodingSize;
            }
        }

        if (exactSizePreset || aspectRatioPreset)
            continue;

        if (requestedWidth && requestedHeight) {
            const auto& minStandardSize = standardVideoSizes()[0];
            if (requestedWidth.value() >= minStandardSize.width() && requestedHeight.value() >= minStandardSize.height()) {
                if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
                    resizePreset = &preset.get();
                    resizeSize = { requestedWidth.value(), requestedHeight.value() };
                }
            }
        } else {
            for (auto& standardSize : standardVideoSizes()) {
                if (standardSize.width() > preset->size.width() || standardSize.height() > preset->size.height())
                    break;
                if ((requestedWidth && requestedWidth.value() != standardSize.width()) || (requestedHeight && requestedHeight.value() != standardSize.height()))
                    continue;

                if (!resizePreset || shouldUsePreset(*resizePreset, preset)) {
                    resizePreset = &preset.get();
                    resizeSize = standardSize;
                }
            }
        }
    }

    if (!exactSizePreset && !aspectRatioPreset && !resizePreset)
        return { };

    result.requestedFrameRate = requestedFrameRate.value();
    if (exactSizePreset) {
        result.encodingPreset = exactSizePreset;
        result.requestedSize = exactSizePreset->size;
        return result;
    }

    if (aspectRatioPreset) {
        result.encodingPreset = aspectRatioPreset;
        result.requestedSize = aspectRatioMatchSize;
        return result;
    }

    result.encodingPreset = resizePreset;
    result.requestedSize = resizeSize;
    return result;
}

void RealtimeVideoSource::setSizeAndFrameRate(Optional<int> width, Optional<int> height, Optional<double> frameRate)
{
    Optional<RealtimeVideoSource::CaptureSizeAndFrameRate> match;

    auto size = this->size();
    if (!width && !height && !size.isEmpty()) {
        width = size.width();
        height = size.height();
    }

    match = bestSupportedSizeAndFrameRate(width, height, frameRate);
    ASSERT(match);
    if (!match)
        return;

    setSizeAndFrameRateWithPreset(match->requestedSize, match->requestedFrameRate, match->encodingPreset);

    if (!match->requestedSize.isEmpty())
        setSize(match->requestedSize);
    setFrameRate(match->requestedFrameRate);
}

void RealtimeVideoSource::dispatchMediaSampleToObservers(MediaSample& sample)
{
    MediaTime sampleTime = sample.outputPresentationTime();
    if (!sampleTime || !sampleTime.isValid())
        sampleTime = sample.presentationTime();

    auto frameTime = sampleTime.toDouble();
    m_observedFrameTimeStamps.append(frameTime);
    m_observedFrameTimeStamps.removeAllMatching([&](auto time) {
        return time <= frameTime - 2;
    });

    auto interval = m_observedFrameTimeStamps.last() - m_observedFrameTimeStamps.first();
    if (interval > 1)
        m_observedFrameRate = (m_observedFrameTimeStamps.size() / interval);

    auto mediaSample = makeRefPtr(&sample);
#if PLATFORM(COCOA)
    if (!isRemote()) {
        auto size = this->size();
        if (!size.isEmpty() && size != expandedIntSize(sample.presentationSize())) {

            if (!m_imageTransferSession || m_imageTransferSession->pixelFormat() != sample.videoPixelFormat())
                m_imageTransferSession = ImageTransferSessionVT::create(sample.videoPixelFormat());

            if (m_imageTransferSession) {
                mediaSample = m_imageTransferSession->convertMediaSample(sample, size);
                if (!mediaSample) {
                    ASSERT_NOT_REACHED();
                    return;
                }
            }
        }
    }
#endif

    videoSampleAvailable(mediaSample.releaseNonNull());
}

} // namespace WebCore

#endif // ENABLE(MEDIA_STREAM)