Added plugin API for outputting sound to an audio device, using a simple CAAudioGraph

Virtual camera output uses above API to output the layout's audio to a user selected system audio device
This commit is contained in:
Zakk 2020-05-22 05:08:11 -04:00
parent 4a79eb3d54
commit 2a2aa7eb91
16 changed files with 303 additions and 8 deletions

View file

@ -263,6 +263,7 @@
34DA1D3E1BF823E700132486 /* CSNewOutputWindowController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DA1D3C1BF823E700132486 /* CSNewOutputWindowController.m */; };
34DA1D3F1BF823E700132486 /* CSNewOutputWindowController.xib in Resources */ = {isa = PBXBuildFile; fileRef = 34DA1D3D1BF823E700132486 /* CSNewOutputWindowController.xib */; };
34DC2FB01B512362008F12A2 /* CSCaptureBase+TimerDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DC2FAF1B512362008F12A2 /* CSCaptureBase+TimerDelegate.m */; };
34DF0885247663CC00DDA606 /* CASimpleOutputGraph.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DF0884247663CC00DDA606 /* CASimpleOutputGraph.m */; };
34DF75581AA272F100DA9FDE /* LayoutRenderer.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DF75571AA272F100DA9FDE /* LayoutRenderer.m */; };
34E26DDC1FF702BD00ACE58B /* CAMultiAudioSilence.m in Sources */ = {isa = PBXBuildFile; fileRef = 34E26DDB1FF702BD00ACE58B /* CAMultiAudioSilence.m */; };
34E26DDF1FF70E0E00ACE58B /* CAMultiAudioGenericOutput.m in Sources */ = {isa = PBXBuildFile; fileRef = 34E26DDE1FF70E0E00ACE58B /* CAMultiAudioGenericOutput.m */; };
@ -1108,6 +1109,10 @@
34DC2F9B1B50FBCD008F12A2 /* CSTimerSourceProtocol.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CSTimerSourceProtocol.h; sourceTree = "<group>"; };
34DC2FAE1B512362008F12A2 /* CSCaptureBase+TimerDelegate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "CSCaptureBase+TimerDelegate.h"; sourceTree = "<group>"; };
34DC2FAF1B512362008F12A2 /* CSCaptureBase+TimerDelegate.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "CSCaptureBase+TimerDelegate.m"; sourceTree = "<group>"; };
34DF0883247663CC00DDA606 /* CASimpleOutputGraph.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = CASimpleOutputGraph.h; path = CAMultiAudio/CASimpleOutputGraph.h; sourceTree = "<group>"; };
34DF0884247663CC00DDA606 /* CASimpleOutputGraph.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; name = CASimpleOutputGraph.m; path = CAMultiAudio/CASimpleOutputGraph.m; sourceTree = "<group>"; };
34DF08A32477AC5900DDA606 /* CSSystemAudioOutput.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = CSSystemAudioOutput.h; path = PluginHeaders/CSSystemAudioOutput.h; sourceTree = "<group>"; };
34DF08A52477B0D500DDA606 /* CSSystemAudioNode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = CSSystemAudioNode.h; path = PluginHeaders/CSSystemAudioNode.h; sourceTree = "<group>"; };
34DF75561AA272F100DA9FDE /* LayoutRenderer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LayoutRenderer.h; sourceTree = "<group>"; };
34DF75571AA272F100DA9FDE /* LayoutRenderer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = LayoutRenderer.m; sourceTree = "<group>"; };
34E26DDA1FF702BD00ACE58B /* CAMultiAudioSilence.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = CAMultiAudioSilence.h; path = CAMultiAudio/CAMultiAudioSilence.h; sourceTree = "<group>"; };
@ -1886,6 +1891,8 @@
3408234619BC50AD00CD1F5F /* CSNotifications.h */,
34DBBAB2201412FC00B3DFFD /* CSOutputBase.h */,
34A77454201449450036F8B5 /* CSStreamServiceBase.h */,
34DF08A32477AC5900DDA606 /* CSSystemAudioOutput.h */,
34DF08A52477B0D500DDA606 /* CSSystemAudioNode.h */,
);
name = PluginHeaders;
sourceTree = "<group>";
@ -1973,6 +1980,8 @@
3488161923FE9BBF00FF4E1B /* CAMultiAudioConnection.m */,
3488161E2401010B00FF4E1B /* CAMultiAudioOutputTrackConnection.h */,
3488161F2401010B00FF4E1B /* CAMultiAudioOutputTrackConnection.m */,
34DF0883247663CC00DDA606 /* CASimpleOutputGraph.h */,
34DF0884247663CC00DDA606 /* CASimpleOutputGraph.m */,
);
name = CAMultiAudio;
sourceTree = "<group>";
@ -3140,6 +3149,7 @@
34ED8C551B07371C002C0674 /* MIKMIDIControlChangeCommand.m in Sources */,
345BECFB1F455AF700B46F29 /* CSCIFilterLayoutTransitionViewController.m in Sources */,
34F80D6D1EDBC8C800D890D3 /* OutputDestination+ScriptingAdditions.m in Sources */,
34DF0885247663CC00DDA606 /* CASimpleOutputGraph.m in Sources */,
3494DF381CCD2DB000E921BF /* TPCircularBuffer.c in Sources */,
349461681ABC57C100F28883 /* CSAnimationItem.m in Sources */,
347DBC9D1FF22C0100B98D5E /* CSAppleHEVCCompressor.m in Sources */,

View file

@ -26,6 +26,7 @@
@property (weak) CAMultiAudioEngine *engine;
@property (assign) bool running;
@property (strong) CAMultiAudioDevice *outputNode;
@property (assign) bool isSimple;
-(instancetype)initWithFormat:(AVAudioFormat *)format;

View file

@ -152,6 +152,11 @@
{
if (self.graph.isSimple)
{
return YES;
}
if (!self.converterNode)
{
self.converterNode = [[CAMultiAudioConverter alloc] init];
@ -226,6 +231,11 @@
-(void)setupEffectsChain
{
if (self.graph.isSimple)
{
return;
}
[self setupGraph];
[super setupEffectsChain];
[self setupDownmixer];
@ -234,6 +244,11 @@
-(void)removeEffectsChain
{
if (self.graph.isSimple)
{
return;
}
[self teardownGraph];
[super removeEffectsChain];
}

View file

@ -0,0 +1,38 @@
//
// CASimpleOutputGraph.h
// CocoaSplit
//
// Created by Zakk on 5/21/20.
// Copyright © 2020 Zakk. All rights reserved.
//
/*
This class provides a very simple CAMultiAudioGraph, designed for outputing a single PCM audio stream to an output device
The graph is just: PCMPlayer -> Mixer (for volume control) -> HAL output
*/
#import <Foundation/Foundation.h>
#import "CAMultiAudio.h"
NS_ASSUME_NONNULL_BEGIN
@interface CASimpleOutputGraph : NSObject
{
CAMultiAudioGraph *_audioGraph;
CAMultiAudioMixer *_audioMixer;
CAMultiAudioPCMPlayer *_player;
}
@property (strong) CAMultiAudioDevice *outputNode;
-(instancetype) initWithAudioFormat:(AVAudioFormat *)audioFormat withOutputNode:(CAMultiAudioDevice *)outputNode;
-(void) playSampleBuffer:(CMSampleBufferRef)sampleBuffer;
-(void)start;
-(void)stop;
@end
NS_ASSUME_NONNULL_END

View file

@ -0,0 +1,75 @@
//
// CASimpleOutputGraph.m
// CocoaSplit
//
// Created by Zakk on 5/21/20.
// Copyright © 2020 Zakk. All rights reserved.
//
#import "CASimpleOutputGraph.h"
@implementation CASimpleOutputGraph
@synthesize outputNode = _outputNode;
-(instancetype)initWithAudioFormat:(AVAudioFormat *)audioFormat withOutputNode:(CAMultiAudioDevice *)outputNode
{
if (self = [self init])
{
self.outputNode = outputNode;
[self buildGraph:audioFormat];
}
return self;
}
-(void)buildGraph:(AVAudioFormat *)audioFormat
{
_audioGraph = [[CAMultiAudioGraph alloc] initWithFormat:audioFormat];
_audioGraph.isSimple = YES;
_audioMixer = [[CAMultiAudioMixer alloc] init];
_player = [[CAMultiAudioPCMPlayer alloc] init];
_player.inputFormat = audioFormat;
if (self.outputNode)
{
[_audioGraph addNode:self.outputNode];
_audioGraph.outputNode = self.outputNode;
[_audioGraph.outputNode setOutputForDevice];
}
[_audioGraph addNode:_audioMixer];
[_audioGraph addNode:_player];
[_audioGraph connectNode:_audioMixer toNode:_audioGraph.outputNode];
[_audioGraph connectNode:_player toNode:_audioMixer];
_audioMixer.volume = 1.0f;
[self start];
}
-(void)playSampleBuffer:(CMSampleBufferRef)sampleBuffer
{
if (_player)
{
[_player scheduleBuffer:sampleBuffer];
}
}
-(void)start
{
if (_audioGraph)
{
[_audioGraph startGraph];
_player.enabled = YES;
}
}
-(void)stop
{
if (_audioGraph)
{
[_audioGraph stopGraph];
}
}
@end

View file

@ -13,6 +13,7 @@
#import "CSPcmPlayer.h"
#import "AppDelegate.h"
#import "PreviewView.h"
#import "CASimpleOutputGraph.h"
@implementation CSPluginServices
@ -191,5 +192,17 @@
return [[CSOauth2Authenticator alloc] initWithServiceName:serviceName clientID:client_id flowType:flow_type config:config_dict];
}
-(NSArray *)audioOutputs
{
return [[CAMultiAudioDevice allDevices] filteredArrayUsingPredicate:[NSPredicate predicateWithFormat:@"hasOutput == YES"]];
}
-(CSSystemAudioOutput *)systemAudioOutputForFormat:(AVAudioFormat *)audioFormat forDevice:(CSSystemAudioNode *)device
{
CASimpleOutputGraph *systemOutput = [[CASimpleOutputGraph alloc] initWithAudioFormat:audioFormat withOutputNode:(CAMultiAudioDevice *)device];
return (CSSystemAudioOutput *)systemOutput;
}
@end

View file

@ -9,6 +9,8 @@
#import <Foundation/Foundation.h>
#import "CSPcmPlayer.h"
#import "CSOauth2Authenticator.h"
#import "CSSystemAudioOutput.h"
#import "CSSystemAudioNode.h"
#import <JavaScriptCore/JavaScriptCore.h>
@ -31,7 +33,8 @@
-(JSValue *)runJavascript:(NSString *)script;
-(NSString *)generateUUID;
-(NSDate *)streamStartDate;
-(NSArray *)audioOutputs;
-(CSSystemAudioOutput *)systemAudioOutputForFormat:(AVAudioFormat *)audioFormat forDevice:(CSSystemAudioOutput *)device;
@property (readonly) double currentFPS;

View file

@ -0,0 +1,19 @@
//
// CSSystemAudioNode.h
// CocoaSplit
//
// Created by Zakk on 5/22/20.
// Copyright © 2020 Zakk. All rights reserved.
//
#ifndef CSSystemAudioNode_h
#define CSSystemAudioNode_h
@interface CSSystemAudioNode : NSObject
@property (strong) NSString *name;
@property (assign) UInt32 deviceID;
@property (strong) NSString *deviceUID;
@end
#endif /* CSSystemAudioNode_h */

View file

@ -0,0 +1,23 @@
//
// CSSystemAudioOutput.h
// CocoaSplit
//
// Created by Zakk on 5/22/20.
// Copyright © 2020 Zakk. All rights reserved.
//
#ifndef CSSystemAudioOutput_h
#define CSSystemAudioOutput_h
#import <CoreMedia/CoreMedia.h>
#import "CSSystemAudioNode.h"
#import <AVFoundation/AVFoundation.h>
@interface CSSystemAudioOutput : NSObject
-(instancetype)initWithAudioFormat:(AVAudioFormat *)audioFormat withOutputNode:(CSSystemAudioNode *)outputNode;
-(void)playSampleBuffer:(CMSampleBufferRef)sampleBuffer;
-(void)start;
-(void)stop;
@end
#endif /* CSSystemAudioOutput_h */

View file

@ -9,17 +9,20 @@
#import <Foundation/Foundation.h>
#import "CSOutputBase.h"
#import "CSVirtualCameraDevice.h"
#import "CSSystemAudioOutput.h"
NS_ASSUME_NONNULL_BEGIN
@interface CSVirtualCameraOutput : CSOutputBase
{
CSSystemAudioOutput *_audioOutput;
}
@property (strong) CSVirtualCameraDevice *cameraDevice;
@property (strong) NSString *deviceName;
@property (assign) bool persistDevice;
@property (strong) NSNumber *pixelFormat;
@property (strong) CSSystemAudioOutput *audioOutputDevice;
@end

View file

@ -7,7 +7,9 @@
//
#import "CSVirtualCameraOutput.h"
#import "CSPluginServices.h"
#import <AVFoundation/AVFoundation.h>
@implementation CSVirtualCameraOutput
@ -26,7 +28,7 @@
self.cameraDevice.name = self.deviceName;
self.cameraDevice.persistOnDisconnect = self.persistDevice;
self.cameraDevice.deviceUID = self.cameraDevice.name;
self.cameraDevice.deviceUID = [NSString stringWithFormat:@"0x145424105986211e"];
self.cameraDevice.frameRate = 1.0f/CMTimeGetSeconds(frameData.videoDuration);
self.cameraDevice.width = CVPixelBufferGetWidth(useImage);
self.cameraDevice.height = CVPixelBufferGetHeight(useImage);
@ -37,14 +39,50 @@
self.cameraDevice.pixelFormat = kCVPixelFormatType_32BGRA;
}
[self.cameraDevice createDeviceWithCompletionBlock:nil];
return NO; //We'll start next frame or so
} else if (self.cameraDevice.isReady) {
[self.cameraDevice publishCVPixelBufferFrame:useImage];
return YES;
}
return NO;
if (self.audioOutputDevice)
{
NSString *audioTrackkey = nil;
if (self.activeAudioTracks && (self.activeAudioTracks.allKeys.count > 0))
{
audioTrackkey = self.activeAudioTracks.allKeys.firstObject;
}
if (!audioTrackkey)
{
audioTrackkey = frameData.pcmAudioSamples.allKeys.firstObject;
}
NSArray *pcmSamples = frameData.pcmAudioSamples[audioTrackkey];
for (id object in pcmSamples)
{
CMSampleBufferRef audioSample = (__bridge CMSampleBufferRef)object;
CMFormatDescriptionRef sampleAudioFormat = CMSampleBufferGetFormatDescription(audioSample);
AVAudioFormat *audioFormat = [[AVAudioFormat alloc] initWithCMAudioFormatDescription:sampleAudioFormat];
if (!_audioOutput)
{
_audioOutput = [CSPluginServices.sharedPluginServices systemAudioOutputForFormat:audioFormat forDevice:self.audioOutputDevice];
[_audioOutput start];
}
if (!_audioOutput)
{
break;
}
[_audioOutput playSampleBuffer:audioSample];
}
}
return YES;
}
-(void)dealloc

View file

@ -9,6 +9,8 @@
#import <Foundation/Foundation.h>
#import "CSStreamServiceBase.h"
#import "CSVirtualCameraOutput.h"
#import "CSSystemAudioNode.h"
NS_ASSUME_NONNULL_BEGIN
@interface CSVirtualCameraOutputService : CSStreamServiceBase
@ -16,7 +18,9 @@ NS_ASSUME_NONNULL_BEGIN
@property (strong) NSString *layoutName;
@property (strong) NSString *deviceName;
@property (strong) NSNumber *pixelFormat;
@property (strong) CSSystemAudioNode *audioOutput;
@property (assign) bool persistDevice;
@property (strong) NSString *audioOutputDeviceUID;
@end

View file

@ -30,6 +30,7 @@
self.output.deviceName = [self getServiceDestination];
self.output.persistDevice = self.persistDevice;
self.output.pixelFormat = self.pixelFormat;
self.output.audioOutputDevice = self.audioOutput;
return self.output;
}
@ -66,6 +67,7 @@
[aCoder encodeObject:self.deviceName forKey:@"deviceName"];
[aCoder encodeBool:self.persistDevice forKey:@"persistDevice"];
[aCoder encodeObject:self.pixelFormat forKey:@"pixelFormat"];
[aCoder encodeObject:self.audioOutput.deviceUID forKey:@"audioOutputDeviceUID"];
}
@ -76,6 +78,7 @@
self.deviceName = [aDecoder decodeObjectForKey:@"deviceName"];
self.persistDevice = [aDecoder decodeObjectForKey:@"persistDevice"];
self.pixelFormat = [aDecoder decodeObjectForKey:@"pixelFormat"];
self.audioOutputDeviceUID = [aDecoder decodeObjectForKey:@"audioOutputDeviceUID"];
}
return self;

View file

@ -18,6 +18,8 @@ NS_ASSUME_NONNULL_BEGIN
@property (weak) CSVirtualCameraOutputService *serviceObj;
@property (strong) NSDictionary *pixelFormats;
@property (strong) NSArray *formatSortDescriptors;
@property (strong) NSArray *audioOutputs;
@end
NS_ASSUME_NONNULL_END

View file

@ -7,6 +7,7 @@
//
#import "CSVirtualCameraOutputViewController.h"
#import "CSPluginServices.h"
@interface CSVirtualCameraOutputViewController ()
@ -26,6 +27,19 @@
@"Component Y'CbCr 8-bit 4:2:2 (yuvs)": @(kCVPixelFormatType_422YpCbCr8_yuvs)
};
self.formatSortDescriptors = @[[NSSortDescriptor sortDescriptorWithKey:@"key" ascending:YES]];
self.audioOutputs = [[CSPluginServices sharedPluginServices] audioOutputs];
if (self.serviceObj.audioOutputDeviceUID)
{
for (CSSystemAudioNode *aNode in self.audioOutputs)
{
if ([aNode.deviceUID isEqualToString:self.serviceObj.audioOutputDeviceUID])
{
self.serviceObj.audioOutput = aNode;
self.serviceObj.audioOutputDeviceUID = aNode.deviceUID;
break;
}
}
}
}

View file

@ -38,7 +38,7 @@
</connections>
</textField>
<button verticalHuggingPriority="750" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="gqU-gi-tWj">
<rect key="frame" x="-3" y="151" width="109" height="20"/>
<rect key="frame" x="-3" y="99" width="109" height="20"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMinY="YES"/>
<buttonCell key="cell" type="check" title="Persist on exit" bezelStyle="regularSquare" imagePosition="trailing" controlSize="small" state="on" inset="2" id="eKG-XL-xsx">
<behavior key="behavior" changeContents="YES" doesNotDimImage="YES" lightByContents="YES"/>
@ -73,18 +73,47 @@
</popUpButtonCell>
<connections>
<binding destination="cNV-F2-BRe" name="content" keyPath="arrangedObjects" id="Njd-be-fAO"/>
<binding destination="cNV-F2-BRe" name="contentObjects" keyPath="arrangedObjects.value" previousBinding="Njd-be-fAO" id="gzX-ra-5WR"/>
<binding destination="cNV-F2-BRe" name="contentValues" keyPath="arrangedObjects.key" previousBinding="gzX-ra-5WR" id="AWB-3K-3Ey"/>
<binding destination="cNV-F2-BRe" name="contentObjects" keyPath="arrangedObjects.value" previousBinding="Njd-be-fAO" id="gzX-ra-5WR"/>
<binding destination="nwA-xG-m6e" name="selectedObject" keyPath="selection.pixelFormat" previousBinding="AWB-3K-3Ey" id="Raw-tR-Bk9"/>
</connections>
</popUpButton>
<popUpButton verticalHuggingPriority="750" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="mMq-73-3rJ">
<rect key="frame" x="94" y="173" width="204" height="22"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMinY="YES"/>
<popUpButtonCell key="cell" type="push" title="Item 1" bezelStyle="rounded" alignment="left" controlSize="small" lineBreakMode="truncatingTail" state="on" borderStyle="borderAndBezel" imageScaling="proportionallyDown" inset="2" selectedItem="yL7-pw-YPP" id="304-p7-Gy5">
<behavior key="behavior" lightByBackground="YES" lightByGray="YES"/>
<font key="font" metaFont="controlContent" size="11"/>
<menu key="menu" id="GRe-hY-qQw">
<items>
<menuItem title="Item 1" state="on" id="yL7-pw-YPP"/>
<menuItem title="Item 2" id="sz6-qg-NMU"/>
<menuItem title="Item 3" id="8gk-m1-kKV"/>
</items>
</menu>
</popUpButtonCell>
<connections>
<binding destination="GXP-21-98B" name="content" keyPath="arrangedObjects" id="RPq-Ve-r9B"/>
<binding destination="GXP-21-98B" name="contentValues" keyPath="arrangedObjects.name" previousBinding="RPq-Ve-r9B" id="LhQ-An-2CA"/>
<binding destination="nwA-xG-m6e" name="selectedObject" keyPath="selection.audioOutput" previousBinding="LhQ-An-2CA" id="xIM-jC-Xf7"/>
</connections>
</popUpButton>
<textField horizontalHuggingPriority="251" verticalHuggingPriority="750" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="J4e-QE-Dz5">
<rect key="frame" x="-2" y="178" width="74" height="14"/>
<autoresizingMask key="autoresizingMask"/>
<textFieldCell key="cell" controlSize="small" lineBreakMode="clipping" title="Audio Output" id="f3R-qt-s4U">
<font key="font" metaFont="controlContent" size="11"/>
<color key="textColor" name="labelColor" catalog="System" colorSpace="catalog"/>
<color key="backgroundColor" name="textBackgroundColor" catalog="System" colorSpace="catalog"/>
</textFieldCell>
</textField>
</subviews>
<point key="canvasLocation" x="140" y="154"/>
</customView>
<dictionaryController objectClassName="_NSDictionaryControllerKeyValuePair" id="cNV-F2-BRe" userLabel="PixelFormatsController">
<connections>
<binding destination="-2" name="contentDictionary" keyPath="self.pixelFormats" id="eUP-Tn-gEp"/>
<binding destination="-2" name="sortDescriptors" keyPath="self.formatSortDescriptors" id="gfW-Yl-5kV"/>
<binding destination="-2" name="contentDictionary" keyPath="self.pixelFormats" id="eUP-Tn-gEp"/>
</connections>
</dictionaryController>
<objectController id="nwA-xG-m6e">
@ -92,5 +121,10 @@
<binding destination="-2" name="contentObject" keyPath="self.serviceObj" id="jlu-iQ-kTU"/>
</connections>
</objectController>
<arrayController id="GXP-21-98B" userLabel="audioOutputsController">
<connections>
<binding destination="-2" name="contentArray" keyPath="self.audioOutputs" id="pj5-wR-hd9"/>
</connections>
</arrayController>
</objects>
</document>