MediaRecorderPrivateWriterCocoa.mm [plain text]
/*
* Copyright (C) 2018 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "config.h"
#include "MediaRecorderPrivateWriterCocoa.h"
#if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
#include "AudioStreamDescription.h"
#include "Logging.h"
#include "MediaStreamTrackPrivate.h"
#include "WebAudioBufferList.h"
#include <AVFoundation/AVAssetWriter.h>
#include <AVFoundation/AVAssetWriterInput.h>
#include <pal/cf/CoreMediaSoftLink.h>
#include <wtf/FileSystem.h>
#import <pal/cocoa/AVFoundationSoftLink.h>
#undef AVEncoderBitRateKey
#define AVEncoderBitRateKey getAVEncoderBitRateKeyWithFallback()
#undef AVFormatIDKey
#define AVFormatIDKey getAVFormatIDKeyWithFallback()
#undef AVNumberOfChannelsKey
#define AVNumberOfChannelsKey getAVNumberOfChannelsKeyWithFallback()
#undef AVSampleRateKey
#define AVSampleRateKey getAVSampleRateKeyWithFallback()
namespace WebCore {
using namespace PAL;
static NSString *getAVFormatIDKeyWithFallback()
{
if (PAL::canLoad_AVFoundation_AVFormatIDKey())
return PAL::get_AVFoundation_AVFormatIDKey();
RELEASE_LOG_ERROR(Media, "Failed to load AVFormatIDKey");
return @"AVFormatIDKey";
}
static NSString *getAVNumberOfChannelsKeyWithFallback()
{
if (PAL::canLoad_AVFoundation_AVNumberOfChannelsKey())
return PAL::get_AVFoundation_AVNumberOfChannelsKey();
RELEASE_LOG_ERROR(Media, "Failed to load AVNumberOfChannelsKey");
return @"AVNumberOfChannelsKey";
}
static NSString *getAVSampleRateKeyWithFallback()
{
if (PAL::canLoad_AVFoundation_AVSampleRateKey())
return PAL::get_AVFoundation_AVSampleRateKey();
RELEASE_LOG_ERROR(Media, "Failed to load AVSampleRateKey");
return @"AVSampleRateKey";
}
static NSString *getAVEncoderBitRateKeyWithFallback()
{
if (PAL::canLoad_AVFoundation_AVEncoderBitRateKey())
return PAL::get_AVFoundation_AVEncoderBitRateKey();
RELEASE_LOG_ERROR(Media, "Failed to load AVEncoderBitRateKey");
return @"AVEncoderBitRateKey";
}
RefPtr<MediaRecorderPrivateWriter> MediaRecorderPrivateWriter::create(const MediaStreamTrackPrivate* audioTrack, const MediaStreamTrackPrivate* videoTrack)
{
NSString *directory = FileSystem::createTemporaryDirectory(@"videos");
NSString *filename = [NSString stringWithFormat:@"/%lld.mp4", CMClockGetTime(CMClockGetHostTimeClock()).value];
NSString *path = [directory stringByAppendingString:filename];
NSURL *outputURL = [NSURL fileURLWithPath:path];
String filePath = [path UTF8String];
NSError *error = nil;
auto avAssetWriter = adoptNS([PAL::allocAVAssetWriterInstance() initWithURL:outputURL fileType:AVFileTypeMPEG4 error:&error]);
if (error) {
RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)error.code);
return nullptr;
}
auto writer = adoptRef(*new MediaRecorderPrivateWriter(WTFMove(avAssetWriter), WTFMove(filePath)));
if (audioTrack && !writer->setAudioInput())
return nullptr;
if (videoTrack) {
auto& settings = videoTrack->settings();
if (!writer->setVideoInput(settings.width(), settings.height()))
return nullptr;
}
return WTFMove(writer);
}
MediaRecorderPrivateWriter::MediaRecorderPrivateWriter(RetainPtr<AVAssetWriter>&& avAssetWriter, String&& filePath)
: m_writer(WTFMove(avAssetWriter))
, m_path(WTFMove(filePath))
{
}
MediaRecorderPrivateWriter::~MediaRecorderPrivateWriter()
{
clear();
}
void MediaRecorderPrivateWriter::clear()
{
if (m_videoInput) {
m_videoInput.clear();
dispatch_release(m_videoPullQueue);
}
if (m_audioInput) {
m_audioInput.clear();
dispatch_release(m_audioPullQueue);
}
if (m_writer)
m_writer.clear();
}
bool MediaRecorderPrivateWriter::setVideoInput(int width, int height)
{
ASSERT(!m_videoInput);
NSDictionary *compressionProperties = @{
AVVideoAverageBitRateKey : [NSNumber numberWithInt:width * height * 12],
AVVideoExpectedSourceFrameRateKey : @(30),
AVVideoMaxKeyFrameIntervalKey : @(120),
AVVideoProfileLevelKey : AVVideoProfileLevelH264MainAutoLevel
};
NSDictionary *videoSettings = @{
AVVideoCodecKey: AVVideoCodecH264,
AVVideoWidthKey: [NSNumber numberWithInt:width],
AVVideoHeightKey: [NSNumber numberWithInt:height],
AVVideoCompressionPropertiesKey: compressionProperties
};
m_videoInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeVideo outputSettings:videoSettings sourceFormatHint:nil]);
[m_videoInput setExpectsMediaDataInRealTime:true];
if (![m_writer canAddInput:m_videoInput.get()]) {
m_videoInput = nullptr;
RELEASE_LOG_ERROR(MediaStream, "the video input is not allowed to add to the AVAssetWriter");
return false;
}
[m_writer addInput:m_videoInput.get()];
m_videoPullQueue = dispatch_queue_create("WebCoreVideoRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
return true;
}
bool MediaRecorderPrivateWriter::setAudioInput()
{
ASSERT(!m_audioInput);
NSDictionary *audioSettings = @{
AVEncoderBitRateKey : @(28000),
AVFormatIDKey : @(kAudioFormatMPEG4AAC),
AVNumberOfChannelsKey : @(1),
AVSampleRateKey : @(22050)
};
m_audioInput = adoptNS([PAL::allocAVAssetWriterInputInstance() initWithMediaType:AVMediaTypeAudio outputSettings:audioSettings sourceFormatHint:nil]);
[m_audioInput setExpectsMediaDataInRealTime:true];
if (![m_writer canAddInput:m_audioInput.get()]) {
m_audioInput = nullptr;
RELEASE_LOG_ERROR(MediaStream, "the audio input is not allowed to add to the AVAssetWriter");
return false;
}
[m_writer addInput:m_audioInput.get()];
m_audioPullQueue = dispatch_queue_create("WebCoreAudioRecordingPullBufferQueue", DISPATCH_QUEUE_SERIAL);
return true;
}
static inline RetainPtr<CMSampleBufferRef> copySampleBufferWithCurrentTimeStamp(CMSampleBufferRef originalBuffer)
{
CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
CMItemCount count = 0;
CMSampleBufferGetSampleTimingInfoArray(originalBuffer, 0, nil, &count);
Vector<CMSampleTimingInfo> timeInfo(count);
CMSampleBufferGetSampleTimingInfoArray(originalBuffer, count, timeInfo.data(), &count);
for (CMItemCount i = 0; i < count; i++) {
timeInfo[i].decodeTimeStamp = kCMTimeInvalid;
timeInfo[i].presentationTimeStamp = startTime;
}
CMSampleBufferRef newBuffer = nullptr;
auto error = CMSampleBufferCreateCopyWithNewTiming(kCFAllocatorDefault, originalBuffer, count, timeInfo.data(), &newBuffer);
if (error)
return nullptr;
return adoptCF(newBuffer);
}
void MediaRecorderPrivateWriter::appendVideoSampleBuffer(CMSampleBufferRef sampleBuffer)
{
ASSERT(m_videoInput);
if (m_isStopped)
return;
if (!m_hasStartedWriting) {
if (![m_writer startWriting]) {
m_isStopped = true;
RELEASE_LOG_ERROR(MediaStream, "create AVAssetWriter instance failed with error code %ld", (long)[m_writer error]);
return;
}
[m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
m_hasStartedWriting = true;
RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
[m_videoInput requestMediaDataWhenReadyOnQueue:m_videoPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
do {
if (![m_videoInput isReadyForMoreMediaData])
break;
auto locker = holdLock(m_videoLock);
if (m_videoBufferPool.isEmpty())
break;
auto buffer = m_videoBufferPool.takeFirst();
locker.unlockEarly();
if (![m_videoInput appendSampleBuffer:buffer.get()])
break;
} while (true);
if (m_isStopped && m_videoBufferPool.isEmpty()) {
[m_videoInput markAsFinished];
m_finishWritingVideoSemaphore.signal();
}
}];
return;
}
auto bufferWithCurrentTime = copySampleBufferWithCurrentTimeStamp(sampleBuffer);
if (!bufferWithCurrentTime)
return;
auto locker = holdLock(m_videoLock);
m_videoBufferPool.append(WTFMove(bufferWithCurrentTime));
}
static inline RetainPtr<CMFormatDescriptionRef> createAudioFormatDescription(const AudioStreamDescription& description)
{
auto basicDescription = WTF::get<const AudioStreamBasicDescription*>(description.platformDescription().description);
CMFormatDescriptionRef format = nullptr;
auto error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, basicDescription, 0, NULL, 0, NULL, NULL, &format);
if (error)
return nullptr;
return adoptCF(format);
}
static inline RetainPtr<CMSampleBufferRef> createAudioSampleBufferWithPacketDescriptions(CMFormatDescriptionRef format, size_t sampleCount)
{
CMTime startTime = CMClockGetTime(CMClockGetHostTimeClock());
CMSampleBufferRef sampleBuffer = nullptr;
auto error = CMAudioSampleBufferCreateWithPacketDescriptions(kCFAllocatorDefault, NULL, false, NULL, NULL, format, sampleCount, startTime, NULL, &sampleBuffer);
if (error)
return nullptr;
return adoptCF(sampleBuffer);
}
void MediaRecorderPrivateWriter::appendAudioSampleBuffer(const PlatformAudioData& data, const AudioStreamDescription& description, const WTF::MediaTime&, size_t sampleCount)
{
ASSERT(m_audioInput);
if ((!m_hasStartedWriting && m_videoInput) || m_isStopped)
return;
auto format = createAudioFormatDescription(description);
if (!format)
return;
if (m_isFirstAudioSample) {
if (!m_videoInput) {
// audio-only recording.
if (![m_writer startWriting]) {
m_isStopped = true;
return;
}
[m_writer startSessionAtSourceTime:CMClockGetTime(CMClockGetHostTimeClock())];
m_hasStartedWriting = true;
}
m_isFirstAudioSample = false;
RefPtr<MediaRecorderPrivateWriter> protectedThis = this;
[m_audioInput requestMediaDataWhenReadyOnQueue:m_audioPullQueue usingBlock:[this, protectedThis = WTFMove(protectedThis)] {
do {
if (![m_audioInput isReadyForMoreMediaData])
break;
auto locker = holdLock(m_audioLock);
if (m_audioBufferPool.isEmpty())
break;
auto buffer = m_audioBufferPool.takeFirst();
locker.unlockEarly();
[m_audioInput appendSampleBuffer:buffer.get()];
} while (true);
if (m_isStopped && m_audioBufferPool.isEmpty()) {
[m_audioInput markAsFinished];
m_finishWritingAudioSemaphore.signal();
}
}];
}
auto sampleBuffer = createAudioSampleBufferWithPacketDescriptions(format.get(), sampleCount);
if (!sampleBuffer)
return;
auto error = CMSampleBufferSetDataBufferFromAudioBufferList(sampleBuffer.get(), kCFAllocatorDefault, kCFAllocatorDefault, 0, downcast<WebAudioBufferList>(data).list());
if (error)
return;
auto locker = holdLock(m_audioLock);
m_audioBufferPool.append(WTFMove(sampleBuffer));
}
void MediaRecorderPrivateWriter::stopRecording()
{
if (m_isStopped)
return;
m_isStopped = true;
if (!m_hasStartedWriting)
return;
ASSERT([m_writer status] == AVAssetWriterStatusWriting);
if (m_videoInput)
m_finishWritingVideoSemaphore.wait();
if (m_audioInput)
m_finishWritingAudioSemaphore.wait();
[m_writer finishWritingWithCompletionHandler:[this, weakPtr = makeWeakPtr(*this)]() mutable {
m_finishWritingSemaphore.signal();
callOnMainThread([this, weakPtr = WTFMove(weakPtr)] {
if (!weakPtr)
return;
m_isStopped = false;
m_hasStartedWriting = false;
m_isFirstAudioSample = true;
clear();
});
}];
m_finishWritingSemaphore.wait();
}
RefPtr<SharedBuffer> MediaRecorderPrivateWriter::fetchData()
{
if ((m_path.isEmpty() && !m_isStopped) || !m_hasStartedWriting)
return nullptr;
return SharedBuffer::createWithContentsOfFile(m_path);
}
} // namespace WebCore
#endif // ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)