From a4d92b33af55168765cd941eab72e1c85af25a33 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 09:56:32 +0200 Subject: [PATCH 01/14] iOS sound + feature device switch for iOS --- Jamulus.pro | 8 +- ios/Info.plist | 4 + ios/ios_app_delegate.mm | 7 +- ios/sound.h | 61 +----- ios/sound.mm | 404 +++++++++++++++++++++++--------------- src/client.cpp | 23 +++ src/client.h | 6 + src/clientdlg.cpp | 11 ++ src/clientdlg.h | 14 ++ src/clientsettingsdlg.cpp | 2 +- src/main.cpp | 1 + src/settings.h | 2 + src/socket.cpp | 37 +++- src/socket.h | 6 +- 14 files changed, 352 insertions(+), 234 deletions(-) diff --git a/Jamulus.pro b/Jamulus.pro index 3394cfe7ab..b0ac5a3998 100644 --- a/Jamulus.pro +++ b/Jamulus.pro @@ -186,12 +186,8 @@ win32 { OBJECTIVE_SOURCES += ios/sound.mm QMAKE_TARGET_BUNDLE_PREFIX = com.jamulussoftware.jamulus QMAKE_APPLICATION_BUNDLE_NAME. = $$TARGET - LIBS += -framework CoreFoundation \ - -framework CoreServices \ - -framework AVFoundation \ - -framework CoreMIDI \ - -framework AudioToolbox \ - -framework Foundation + LIBS += -framework AVFoundation \ + -framework AudioToolbox } else:android { # we want to compile with C++14 CONFIG += c++14 diff --git a/ios/Info.plist b/ios/Info.plist index dfc2d2937c..aa4a9559b8 100644 --- a/ios/Info.plist +++ b/ios/Info.plist @@ -29,6 +29,10 @@ UIApplicationSupportsIndirectInputEvents + UIBackgroundModes + + audio + UILaunchScreen UILaunchStoryboardName diff --git a/ios/ios_app_delegate.mm b/ios/ios_app_delegate.mm index 54b78a2651..42a1dbee3b 100644 --- a/ios/ios_app_delegate.mm +++ b/ios/ios_app_delegate.mm @@ -1,4 +1,3 @@ - #import "ios_app_delegate.h" @interface QIOSApplicationDelegate () @@ -7,10 +6,8 @@ @interface QIOSApplicationDelegate () @implementation QIOSApplicationDelegate - - -- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { - +- (BOOL)application:(UIApplication*)application didFinishLaunchingWithOptions:(NSDictionary*)launchOptions +{ return YES; } @end diff --git a/ios/sound.h b/ios/sound.h index 7f08179867..aeeec03f79 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -1,8 +1,8 @@ /******************************************************************************\ - * Copyright (c) 2004-2020 + * Copyright (c) 2004-2021 * * Author(s): - * ann0see based on code from Volker Fischer + * ann0see and ngocdh based on code from Volker Fischer * ****************************************************************************** * @@ -23,13 +23,12 @@ \******************************************************************************/ #pragma once -#include #include -#include #include "soundbase.h" #include "global.h" -/* Classes ********************************************************************/ +#import + class CSound : public CSoundBase { Q_OBJECT @@ -44,64 +43,20 @@ class CSound : public CSoundBase virtual int Init ( const int iNewPrefMonoBufferSize ); virtual void Start(); virtual void Stop(); + virtual void processBufferList ( AudioBufferList*, CSound* ); + virtual void SetInputDeviceId ( int deviceid ); - // channel selection - virtual int GetNumInputChannels() { return iNumInChanPlusAddChan; } - virtual QString GetInputChannelName ( const int iDiD ) { return sChannelNamesInput[iDiD]; } - virtual void SetLeftInputChannel ( const int iNewChan ); - virtual void SetRightInputChannel ( const int iNewChan ); - virtual int GetLeftInputChannel() { return iSelInputLeftChannel; } - virtual int GetRightInputChannel() { return iSelInputRightChannel; } - - virtual int GetNumOutputChannels() { return iNumOutChan; } - virtual QString GetOutputChannelName ( const int iDiD ) { return sChannelNamesOutput[iDiD]; } - virtual void SetLeftOutputChannel ( const int iNewChan ); - virtual void SetRightOutputChannel ( const int iNewChan ); - virtual int GetLeftOutputChannel() { return iSelOutputLeftChannel; } - virtual int GetRightOutputChannel() { return iSelOutputRightChannel; } + AudioUnit audioUnit; // these variables/functions should be protected but cannot since we want // to access them from the callback function CVector vecsTmpAudioSndCrdStereo; int iCoreAudioBufferSizeMono; int iCoreAudioBufferSizeStereo; - long lCurDev; - int iNumInChan; - int iNumInChanPlusAddChan; // includes additional "added" channels - int iNumOutChan; - int iSelInputLeftChannel; - int iSelInputRightChannel; - int iSelOutputLeftChannel; - int iSelOutputRightChannel; - int iSelInBufferLeft; - int iSelInBufferRight; - int iSelInInterlChLeft; - int iSelInInterlChRight; - int iSelAddInBufferLeft; - int iSelAddInBufferRight; - int iSelAddInInterlChLeft; - int iSelAddInInterlChRight; - int iSelOutBufferLeft; - int iSelOutBufferRight; - int iSelOutInterlChLeft; - int iSelOutInterlChRight; - CVector vecNumInBufChan; - CVector vecNumOutBufChan; + bool isInitialized; protected: - virtual QString LoadAndInitializeDriver ( QString strDriverName, bool ); - - QString CheckDeviceCapabilities ( const int iDriverIdx ); - void GetAvailableInOutDevices(); - - static void callbackMIDI ( const MIDIPacketList* pktlist, void* refCon, void* ); - - // AVAudioSession audioSession; - MIDIPortRef midiInPortRef; - QString sChannelNamesInput[MAX_NUM_IN_OUT_CHANNELS]; - QString sChannelNamesOutput[MAX_NUM_IN_OUT_CHANNELS]; - QMutex Mutex; }; diff --git a/ios/sound.mm b/ios/sound.mm index a768b14876..e827f29256 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -1,8 +1,8 @@ /******************************************************************************\ - * Copyright (c) 2004-2020 + * Copyright (c) 2004-2021 * * Author(s): - * ann0see based on code from Volker Fischer + * ann0see and ngocdh based on code from Volker Fischer * ****************************************************************************** * @@ -24,209 +24,289 @@ #include "sound.h" #include +#define kOutputBus 0 +#define kInputBus 1 -/* Implementation *************************************************************/ -CSound::CSound ( void (*fpNewProcessCallback) ( CVector& psData, void* arg ), - void* arg, - const QString& strMIDISetup, - const bool , - const QString& ) : - CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), - midiInPortRef ( static_cast ( NULL ) ) +void checkStatus ( int status ) { - NSError *audioSessionError = nil; - - [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; - [[AVAudioSession sharedInstance] requestRecordPermission:^(BOOL granted) { - if (granted) { - // ok - } - }]; - [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; + if ( status ) + { + printf ( "Status not 0! %d\n", status ); + } +} - //GetAvailableInOutDevices(); +void checkStatus ( int status, char* s ) +{ + if ( status ) + { + printf ( "Status not 0! %d - %s \n", status, s ); + } } -int CSound::Init ( const int /*iNewPrefMonoBufferSize*/ ) +/** + This callback is called when sound card needs output data to play. And because Jamulus use the same buffer to store input and output data (input is + sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the + output fetched from server to ioData, which will then be played. + */ +static OSStatus recordingCallback ( void* inRefCon, + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ) { - // store buffer size - //iCoreAudioBufferSizeMono = audioSession.IOBufferDuration; + CSound* pSound = static_cast ( inRefCon ); - // init base class - //CSoundBase::Init ( iCoreAudioBufferSizeMono ); + AudioBuffer buffer; - // set internal buffer size value and calculate stereo buffer size - //iCoreAudioBufferSizeStereo = 2 * iCoreAudioBufferSizeMono; + buffer.mNumberChannels = 2; + buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * buffer.mNumberChannels; + buffer.mData = malloc ( buffer.mDataByteSize ); - // create memory for intermediate audio buffer - //vecsTmpAudioSndCrdStereo.Init ( iCoreAudioBufferSizeStereo ); + // Put buffer in a AudioBufferList + AudioBufferList bufferList; + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0] = buffer; - return iCoreAudioBufferSizeMono; -} + // Then: + // Obtain recorded samples -void CSound::Start() -{ - // ?? + OSStatus status; - // call base class - //CSoundBase::Start(); -} + // Calling Unit Render to store input data to bufferList + status = AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); + // checkStatus(status, (char *)" Just called AudioUnitRender "); -void CSound::Stop() -{ - // ?? - // call base class - //CSoundBase::Stop(); -} + // Now, we have the samples we just read sitting in buffers in bufferList + // Process the new data + pSound->processBufferList ( &bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList -void CSound::GetAvailableInOutDevices() -{ - // https://developer.apple.com/documentation/avfoundation/avaudiosession/1616557-availableinputs?language=objc? + // release the malloc'ed data in the buffer we created earlier + free ( bufferList.mBuffers[0].mData ); + + Float32* pData = (Float32*) ( ioData->mBuffers[0].mData ); + + // copy output data + for ( int i = 0; i < pSound->iCoreAudioBufferSizeMono; i++ ) + { + pData[2 * i] = (Float32) pSound->vecsTmpAudioSndCrdStereo[2 * i] / _MAXSHORT; // left + pData[2 * i + 1] = (Float32) pSound->vecsTmpAudioSndCrdStereo[2 * i + 1] / _MAXSHORT; // right + } + return noErr; } -QString CSound::LoadAndInitializeDriver ( QString strDriverName, bool ) +void CSound::processBufferList ( AudioBufferList* inInputData, CSound* pSound ) // got stereo input data { - // get the driver: check if devices are capable - // reload the driver list of available sound devices - //GetAvailableInOutDevices(); - - // find driver index from given driver name - //int iDriverIdx = INVALID_INDEX; // initialize with an invalid index - - /*for ( int i = 0; i < MAX_NUMBER_SOUND_CARDS; i++ ) - { - if ( strDriverName.compare ( strDriverNames[i] ) == 0 ) - { - iDriverIdx = i; - } - } - - // if the selected driver was not found, return an error message - if ( iDriverIdx == INVALID_INDEX ) - { - return tr ( "The current selected audio device is no longer present in the system." ); - } - - // check device capabilities if it fulfills our requirements - const QString strStat = CheckDeviceCapabilities ( iDriverIdx ); - - // check if device is capable and if not the same device is used - if ( strStat.isEmpty() && ( strCurDevName.compare ( strDriverNames[iDriverIdx] ) != 0 ) ) - { - } - // set the left/right in/output channels - - return strStat; - */ - return ""; + QMutexLocker locker ( &pSound->MutexAudioProcessCallback ); + Float32* pData = static_cast ( inInputData->mBuffers[0].mData ); + + // copy input data + for ( int i = 0; i < pSound->iCoreAudioBufferSizeMono; i++ ) + { + // copy left and right channels separately + pSound->vecsTmpAudioSndCrdStereo[2 * i] = (short) ( pData[2 * i] * _MAXSHORT ); // left + pSound->vecsTmpAudioSndCrdStereo[2 * i + 1] = (short) ( pData[2 * i + 1] * _MAXSHORT ); // right + } + pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); } -QString CSound::CheckDeviceCapabilities ( const int iDriverIdx ) +/* Implementation *************************************************************/ +CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), + void* arg, + const QString& strMIDISetup, + const bool, + const QString& ) : + CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), + midiInPortRef ( static_cast ( NULL ) ) { - /*// This function checks if our required input/output channel - // properties are supported by the selected device. If the return - // string is empty, the device can be used, otherwise the error - // message is returned. - - // check sample rate of input - audioSession.setPreferredSampleRate( SYSTEM_SAMPLE_RATE_HZ ); - // if false, try to set it - if ( audioSession.sampleRate != SYSTEM_SAMPLE_RATE_HZ ) - { - throw CGenErr ( tr ("Could not set sample rate. Please close other applications playing audio.") ); - } - - // special case with 4 input channels: support adding channels - if ( ( lNumInChan == 4 ) && bInputChMixingSupported ) + try { - // add four mixed channels (i.e. 4 normal, 4 mixed channels) - lNumInChanPlusAddChan = 8; - - for ( int iCh = 0; iCh < lNumInChanPlusAddChan; iCh++ ) - { - int iSelCH, iSelAddCH; - - GetSelCHAndAddCH ( iCh, lNumInChan, iSelCH, iSelAddCH ); - - if ( iSelAddCH >= 0 ) - { - // for mixed channels, show both audio channel names to be mixed - channelInputName[iCh] = - channelInputName[iSelCH] + " + " + channelInputName[iSelAddCH]; - } - } + NSError* audioSessionError = nil; + + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; + [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { + if ( granted ) + { + // ok + } + }]; + [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } - else + catch ( const CGenErr& generr ) { - // regular case: no mixing input channels used - lNumInChanPlusAddChan = lNumInChan; + qDebug ( "Sound exception ...." ); // This try-catch seems to fix Connect button crash } -*/ - // everything is ok, return empty string for "no error" case - return ""; + isInitialized = false; } -void CSound::SetLeftInputChannel ( const int iNewChan ) +int CSound::Init ( const int iCoreAudioBufferSizeMono ) { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < iNumInChanPlusAddChan ) ) + try { - iSelInputLeftChannel = iNewChan; + // printf("Init sound ..."); <= to check the number of Sound inits at launch + // init base class + // CSoundBase::Init ( iCoreAudioBufferSizeMono ); this does nothing + this->iCoreAudioBufferSizeMono = iCoreAudioBufferSizeMono; + + // set internal buffer size value and calculate stereo buffer size + iCoreAudioBufferSizeStereo = 2 * iCoreAudioBufferSizeMono; + + // create memory for intermediate audio buffer + vecsTmpAudioSndCrdStereo.Init ( iCoreAudioBufferSizeStereo ); + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + // we are going to play and record so we pick that category + NSError* error = nil; + [sessionInstance setCategory:AVAudioSessionCategoryPlayAndRecord + withOptions:AVAudioSessionCategoryOptionMixWithOthers | AVAudioSessionCategoryOptionDefaultToSpeaker | + AVAudioSessionCategoryOptionAllowBluetooth | AVAudioSessionCategoryOptionAllowBluetoothA2DP | + AVAudioSessionCategoryOptionAllowAirPlay + error:&error]; + + // NGOCDH - using values from jamulus settings 64 = 2.67ms/2 + NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math + [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; + + // set the session's sample rate 48000 - the only supported by Jamulus ? + [sessionInstance setPreferredSampleRate:48000 error:&error]; + [[AVAudioSession sharedInstance] setActive:YES error:&error]; + + OSStatus status; + + // Describe audio component + AudioComponentDescription desc; + desc.componentType = kAudioUnitType_Output; + desc.componentSubType = kAudioUnitSubType_RemoteIO; + desc.componentFlags = 0; + desc.componentFlagsMask = 0; + desc.componentManufacturer = kAudioUnitManufacturer_Apple; + + // Get component + AudioComponent inputComponent = AudioComponentFindNext ( NULL, &desc ); + + // Get audio units + status = AudioComponentInstanceNew ( inputComponent, &audioUnit ); + checkStatus ( status ); + + // Enable IO for recording + UInt32 flag = 1; + status = AudioUnitSetProperty ( audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, kInputBus, &flag, sizeof ( flag ) ); + checkStatus ( status ); + + // Enable IO for playback + status = AudioUnitSetProperty ( audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &flag, sizeof ( flag ) ); + checkStatus ( status ); + + // Describe format + AudioStreamBasicDescription audioFormat; + audioFormat.mSampleRate = 48000.00; + audioFormat.mFormatID = kAudioFormatLinearPCM; + audioFormat.mFormatFlags = kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; + audioFormat.mFramesPerPacket = 1; + audioFormat.mChannelsPerFrame = 2; // steoreo, so 2 interleaved channels + audioFormat.mBitsPerChannel = 32; // sizeof float32 + audioFormat.mBytesPerPacket = 8; // (sizeof float32) * 2 channels + audioFormat.mBytesPerFrame = 8; //(sizeof float32) * 2 channels + + // Apply format + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, + kInputBus, + &audioFormat, + sizeof ( audioFormat ) ); + checkStatus ( status ); + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, + kOutputBus, + &audioFormat, + sizeof ( audioFormat ) ); + checkStatus ( status ); + + // Set callback + AURenderCallbackStruct callbackStruct; + callbackStruct.inputProc = recordingCallback; // this is actually the playback callback + callbackStruct.inputProcRefCon = this; + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Global, + kOutputBus, + &callbackStruct, + sizeof ( callbackStruct ) ); + checkStatus ( status ); + + // Initialise + status = AudioUnitInitialize ( audioUnit ); + checkStatus ( status ); + + isInitialized = true; + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Init exception ...." ); // This try-catch seems to fix Connect button crash } - */ -} -void CSound::SetRightInputChannel ( const int iNewChan ) -{ - /* // apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumInChanPlusAddChan ) ) - { - vSelectedInputChannels[1] = iNewChan; - }*/ + return iCoreAudioBufferSizeMono; } -void CSound::SetLeftOutputChannel ( const int iNewChan ) +void CSound::Start() { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumOutChan ) ) - { - vSelectedOutputChannels[0] = iNewChan; - }*/ + // call base class + CSoundBase::Start(); + try + { + OSStatus err = AudioOutputUnitStart ( audioUnit ); + checkStatus ( err ); + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Start exception ...." ); // This try-catch seems to fix Connect button crash + } } -void CSound::SetRightOutputChannel ( const int iNewChan ) +void CSound::Stop() { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumOutChan ) ) - { - vSelectedOutputChannels[1] = iNewChan; - }*/ + try + { + OSStatus err = AudioOutputUnitStop ( audioUnit ); + checkStatus ( err ); + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Stop exception ...." ); // This try-catch seems to fix Connect button crash + } + // call base class + CSoundBase::Stop(); } - -void CSound::callbackMIDI ( const MIDIPacketList* pktlist, - void* refCon, - void* ) +void CSound::SetInputDeviceId ( int deviceid ) { - /* CSound* pSound = static_cast ( refCon ); - - if ( pSound->midiInPortRef != static_cast ( NULL ) ) + try { - MIDIPacket* midiPacket = const_cast ( pktlist->packet ); + NSError* error = nil; + bool builtinmic = true; - for ( unsigned int j = 0; j < pktlist->numPackets; j++ ) + if ( deviceid == 0 ) + builtinmic = false; // try external device + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device + if ( builtinmic ) + { + [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; + } + else { - // copy packet and send it to the MIDI parser - CVector vMIDIPaketBytes ( midiPacket->length ); - for ( int i = 0; i < midiPacket->length; i++ ) - { - vMIDIPaketBytes[i] = static_cast ( midiPacket->data[i] ); - } - pSound->ParseMIDIMessage ( vMIDIPaketBytes ); - - midiPacket = MIDIPacketNext ( midiPacket ); + unsigned long lastInput = sessionInstance.availableInputs.count - 1; + [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; } - }*/ + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound dev change exception ...." ); // This try-catch seems to fix Connect button crash + } } diff --git a/src/client.cpp b/src/client.cpp index 367972abdb..41129e132a 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -52,6 +52,7 @@ CClient::CClient ( const quint16 iPortNumber, bReverbOnLeftChan ( false ), iReverbLevel ( 0 ), iInputBoost ( 1 ), + iBuiltInMicId ( 0 ), iSndCrdPrefFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), iSndCrdFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), bSndCrdConversionBufferRequired ( false ), @@ -767,14 +768,28 @@ void CClient::Init() const int iFraSizeDefault = SYSTEM_FRAME_SIZE_SAMPLES * FRAME_SIZE_FACTOR_DEFAULT; const int iFraSizeSafe = SYSTEM_FRAME_SIZE_SAMPLES * FRAME_SIZE_FACTOR_SAFE; +#if defined( Q_OS_IOS ) + bFraSiFactPrefSupported = true; // to reduce sound init time, because we know it's supported in iOS + bFraSiFactDefSupported = true; + bFraSiFactSafeSupported = true; +#else bFraSiFactPrefSupported = ( Sound.Init ( iFraSizePreffered ) == iFraSizePreffered ); bFraSiFactDefSupported = ( Sound.Init ( iFraSizeDefault ) == iFraSizeDefault ); bFraSiFactSafeSupported = ( Sound.Init ( iFraSizeSafe ) == iFraSizeSafe ); +#endif // translate block size index in actual block size const int iPrefMonoFrameSize = iSndCrdPrefFrameSizeFactor * SYSTEM_FRAME_SIZE_SAMPLES; // get actual sound card buffer size using preferred size + // TODO - iOS needs 1 init only, now: 9 inits at launch <- slow + // Initially, I tried to fix this as follows (inside #ifdef ios tag): + // if ( Sound.isInitialized ) + // iMonoBlockSizeSam = iPrefMonoFrameSize; + // else + // iMonoBlockSizeSam = Sound.Init ( iPrefMonoFrameSize ); + // Problem is legitimate setting changes (buffer size for example). + // so the condition should be something like "if ( Sound.isInitialized and APP_IS_INIALIZING)" iMonoBlockSizeSam = Sound.Init ( iPrefMonoFrameSize ); // Calculate the current sound card frame size factor. In case @@ -1236,3 +1251,11 @@ int CClient::EstimatedOverallDelay ( const int iPingTimeMs ) return MathUtils::round ( fTotalBufferDelayMs + iPingTimeMs ); } + +void CClient::SetInputDeviceId ( const int deviceid ) +{ +#if defined( Q_OS_IOS ) + // iOS only + Sound.SetInputDeviceId ( deviceid ); +#endif +} diff --git a/src/client.h b/src/client.h index 929da28853..dc5d816920 100644 --- a/src/client.h +++ b/src/client.h @@ -240,6 +240,9 @@ class CClient : public QObject void SetInputBoost ( const int iNewBoost ) { iInputBoost = iNewBoost; } + void SetBuiltInMicId ( const int iNewMicId ) { iBuiltInMicId = iNewMicId; } + int GetBuiltInMicId() { return iBuiltInMicId; } + void SetRemoteInfo() { Channel.SetRemoteInfo ( ChannelInfo ); } void CreateChatTextMes ( const QString& strChatText ) { Channel.CreateChatTextMes ( strChatText ); } @@ -264,6 +267,8 @@ class CClient : public QObject Channel.GetBufErrorRates ( vecErrRates, dLimit, dMaxUpLimit ); } + void SetInputDeviceId ( const int deviceid ); // for mobile devices - 0 for external devices + // settings CChannelCoreInfo ChannelInfo; QString strClientName; @@ -324,6 +329,7 @@ class CClient : public QObject int iReverbLevel; CAudioReverb AudioReverb; int iInputBoost; + int iBuiltInMicId; int iSndCrdPrefFrameSizeFactor; int iSndCrdFrameSizeFactor; diff --git a/src/clientdlg.cpp b/src/clientdlg.cpp index b34d9e741c..d4634fc162 100644 --- a/src/clientdlg.cpp +++ b/src/clientdlg.cpp @@ -375,6 +375,17 @@ CClientDlg::CClientDlg ( CClient* pNCliP, pMenu->addMenu ( pEditMenu ); pMenu->addMenu ( new CHelpMenu ( true, this ) ); +#if defined( Q_OS_IOS ) + // Mobile input -------------------------------------------------------------- + QMenu* pMobileMenu = new QMenu ( tr ( "&Mobile input" ), this ); + + pMobileMenu->addAction ( tr ( "Builtin Mic" ), this, SLOT ( setBuiltinMic() ) ); + + pMobileMenu->addAction ( tr ( "Auto" ), this, SLOT ( unsetBuiltinMic() ) ); + + pMenu->addMenu ( pMobileMenu ); +#endif + // Now tell the layout about the menu layout()->setMenuBar ( pMenu ); diff --git a/src/clientdlg.h b/src/clientdlg.h index 4d2ce7e182..dcfb49ddcd 100644 --- a/src/clientdlg.h +++ b/src/clientdlg.h @@ -231,6 +231,20 @@ public slots: void accept() { close(); } // introduced by pljones + void setBuiltinMic() + { +#if defined( Q_OS_IOS ) + pClient->SetInputDeviceId ( 1 ); +#endif + } + + void unsetBuiltinMic() + { +#if defined( Q_OS_IOS ) + pClient->SetInputDeviceId ( 0 ); +#endif + } + signals: void SendTabChange ( int iTabIdx ); }; diff --git a/src/clientsettingsdlg.cpp b/src/clientsettingsdlg.cpp index 4faf0b1884..e0775a7049 100644 --- a/src/clientsettingsdlg.cpp +++ b/src/clientsettingsdlg.cpp @@ -33,7 +33,7 @@ CClientSettingsDlg::CClientSettingsDlg ( CClient* pNCliP, CClientSettings* pNSet setupUi ( this ); #if defined( Q_OS_IOS ) - // IOS needs menu to close + // iOS needs menu to close QMenuBar* pMenu = new QMenuBar ( this ); QAction* action = pMenu->addAction ( tr ( "&Close" ) ); connect ( action, SIGNAL ( triggered() ), this, SLOT ( close() ) ); diff --git a/src/main.cpp b/src/main.cpp index fb601e74bd..025597903d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -539,6 +539,7 @@ int main ( int argc, char** argv ) #else # if defined( Q_OS_IOS ) bUseGUI = true; + bIsClient = true; // Client only - TODO: maybe a switch in interface to change to server? // bUseMultithreading = true; QApplication* pApp = new QApplication ( argc, argv ); diff --git a/src/settings.h b/src/settings.h index 765b6ee115..57e28b76fc 100644 --- a/src/settings.h +++ b/src/settings.h @@ -120,6 +120,7 @@ class CClientSettings : public CSettings vstrIPAddress ( MAX_NUM_SERVER_ADDR_ITEMS, "" ), iNewClientFaderLevel ( 100 ), iInputBoost ( 1 ), + iBuiltInMicId ( 0 ), iSettingsTab ( SETTING_TAB_AUDIONET ), bConnectDlgShowAllMusicians ( true ), eChannelSortType ( ST_NO_SORT ), @@ -152,6 +153,7 @@ class CClientSettings : public CSettings int iNewClientFaderLevel; int iInputBoost; int iSettingsTab; + int iBuiltInMicId; // 0 for external bool bConnectDlgShowAllMusicians; EChSortType eChannelSortType; int iNumMixerPanelRows; diff --git a/src/socket.cpp b/src/socket.cpp index 94d254920f..d8d41645f2 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -28,6 +28,11 @@ /* Implementation *************************************************************/ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) { + // first store parameters, in case reinit is required (mostly for iOS) + iPortNumber_ = iPortNumber; + iQosNumber_ = iQosNumber; + strServerBindIP_ = strServerBindIP; + #ifdef _WIN32 // for the Windows socket usage we have to start it up first @@ -45,6 +50,12 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const const char tos = (char) iQosNumber; // Quality of Service setsockopt ( UdpSocket, IPPROTO_IP, IP_TOS, &tos, sizeof ( tos ) ); +#ifdef Q_OS_IOS + // ignore the broken pipe signal to avoid crash (iOS) + int valueone = 1; + setsockopt ( UdpSocket, SOL_SOCKET, SO_NOSIGPIPE, &valueone, sizeof ( valueone ) ); +#endif + // allocate memory for network receive and send buffer in samples vecbyRecBuf.Init ( MAX_SIZE_BYTES_NETW_BUF ); @@ -115,6 +126,8 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const // Connections ------------------------------------------------------------- // it is important to do the following connections in this class since we // have a thread transition + if ( bIsInitRan ) + return; // we have different connections for client and server if ( bIsClient ) @@ -142,6 +155,8 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); } + + bIsInitRan = true; // QObject::connect once only } void CSocket::Close() @@ -188,12 +203,22 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr UdpSocketOutAddr.sin_port = htons ( HostAddr.iPort ); UdpSocketOutAddr.sin_addr.s_addr = htonl ( HostAddr.InetAddr.toIPv4Address() ); - sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - (sockaddr*) &UdpSocketOutAddr, - sizeof ( sockaddr_in ) ); + if ( sendto ( UdpSocket, + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + (sockaddr*) &UdpSocketOutAddr, + sizeof ( sockaddr_in ) ) < 0 ) + { + // qDebug("Socket send exception - mostly happens in iOS when returning from idle"); + Init ( iPortNumber_, iQosNumber_, strServerBindIP_ ); // reinit + sendto ( UdpSocket, + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + (sockaddr*) &UdpSocketOutAddr, + sizeof ( sockaddr_in ) ); + } } } diff --git a/src/socket.h b/src/socket.h index bbf6c49880..2a248835bd 100644 --- a/src/socket.h +++ b/src/socket.h @@ -77,7 +77,11 @@ class CSocket : public QObject void Close(); protected: - void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); + void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); + quint16 iPortNumber_; + quint16 iQosNumber_; + QString strServerBindIP_; + bool bIsInitRan; #ifdef _WIN32 SOCKET UdpSocket; From f414a1ce79df3e20f8eaa83ceae1caa04e7f6f35 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 14:49:00 +0200 Subject: [PATCH 02/14] iOS code review modified: ios/sound.h modified: ios/sound.mm modified: src/socket.cpp modified: src/socket.h --- ios/sound.h | 11 +++--- ios/sound.mm | 98 ++++++++++++++++++++++---------------------------- src/socket.cpp | 87 +++++++++++++++++++++++--------------------- src/socket.h | 23 +++--------- 4 files changed, 101 insertions(+), 118 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index aeeec03f79..c44b543081 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -23,7 +23,6 @@ \******************************************************************************/ #pragma once -#include #include "soundbase.h" #include "global.h" @@ -56,7 +55,11 @@ class CSound : public CSoundBase bool isInitialized; protected: - MIDIPortRef midiInPortRef; - - QMutex Mutex; + void checkStatus ( int status ); + static OSStatus recordingCallback ( void* inRefCon, + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ); }; diff --git a/ios/sound.mm b/ios/sound.mm index e827f29256..9d315ef376 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -27,28 +27,39 @@ #define kOutputBus 0 #define kInputBus 1 -void checkStatus ( int status ) +/* Implementation *************************************************************/ +CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), + void* arg, + const QString& strMIDISetup, + const bool, + const QString& ) : + CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), + isInitialized ( false ) { - if ( status ) + try { - printf ( "Status not 0! %d\n", status ); - } -} + NSError* audioSessionError = nil; -void checkStatus ( int status, char* s ) -{ - if ( status ) + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; + [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { + if ( granted ) + { + // ok + } + }]; + [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; + } + catch ( const CGenErr& generr ) { - printf ( "Status not 0! %d - %s \n", status, s ); + qDebug ( "Sound exception ...." ); } } /** - This callback is called when sound card needs output data to play. And because Jamulus use the same buffer to store input and output data (input is - sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the - output fetched from server to ioData, which will then be played. + This callback is called when sound card needs output data to play. + And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. */ -static OSStatus recordingCallback ( void* inRefCon, +OSStatus CSound::recordingCallback ( void* inRefCon, AudioUnitRenderActionFlags* ioActionFlags, const AudioTimeStamp* inTimeStamp, UInt32 inBusNumber, @@ -69,14 +80,10 @@ static OSStatus recordingCallback ( void* inRefCon, bufferList.mNumberBuffers = 1; bufferList.mBuffers[0] = buffer; - // Then: // Obtain recorded samples - OSStatus status; - // Calling Unit Render to store input data to bufferList - status = AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); - // checkStatus(status, (char *)" Just called AudioUnitRender "); + AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); // Now, we have the samples we just read sitting in buffers in bufferList // Process the new data @@ -112,42 +119,12 @@ static OSStatus recordingCallback ( void* inRefCon, pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); } -/* Implementation *************************************************************/ -CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), - void* arg, - const QString& strMIDISetup, - const bool, - const QString& ) : - CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), - midiInPortRef ( static_cast ( NULL ) ) -{ - try - { - NSError* audioSessionError = nil; - - [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; - [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { - if ( granted ) - { - // ok - } - }]; - [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; - } - catch ( const CGenErr& generr ) - { - qDebug ( "Sound exception ...." ); // This try-catch seems to fix Connect button crash - } - isInitialized = false; -} - +// TODO - CSound::Init is called multiple times at launch to verify device capabilities. +// For iOS though, Init takes long, so should better reduce those inits for iOS (Now: 9 times/launch). int CSound::Init ( const int iCoreAudioBufferSizeMono ) { try { - // printf("Init sound ..."); <= to check the number of Sound inits at launch - // init base class - // CSoundBase::Init ( iCoreAudioBufferSizeMono ); this does nothing this->iCoreAudioBufferSizeMono = iCoreAudioBufferSizeMono; // set internal buffer size value and calculate stereo buffer size @@ -166,11 +143,11 @@ static OSStatus recordingCallback ( void* inRefCon, AVAudioSessionCategoryOptionAllowAirPlay error:&error]; - // NGOCDH - using values from jamulus settings 64 = 2.67ms/2 + // using values from jamulus settings 64 = 2.67ms/2 NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; - // set the session's sample rate 48000 - the only supported by Jamulus ? + // set the session's sample rate 48000 - the only supported by Jamulus [sessionInstance setPreferredSampleRate:48000 error:&error]; [[AVAudioSession sharedInstance] setActive:YES error:&error]; @@ -247,7 +224,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Init exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Init exception ...." ); } return iCoreAudioBufferSizeMono; @@ -264,7 +241,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Start exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Start exception ...." ); } } @@ -277,7 +254,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Stop exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Stop exception ...." ); } // call base class CSoundBase::Stop(); @@ -294,6 +271,7 @@ static OSStatus recordingCallback ( void* inRefCon, builtinmic = false; // try external device AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device if ( builtinmic ) { @@ -307,6 +285,14 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound dev change exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound dev change exception ...." ); + } +} + +void CSound::checkStatus ( int status ) +{ + if ( status ) + { + printf ( "Status not 0! %d\n", status ); } } diff --git a/src/socket.cpp b/src/socket.cpp index d8d41645f2..cff7ba3549 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -26,12 +26,54 @@ #include "server.h" /* Implementation *************************************************************/ -void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) + +// Connections ------------------------------------------------------------- +// it is important to do the following connections in this class since we +// have a thread transition + +// we have different connections for client and server, created after Init in corresponding constructor + +CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pChannel ( pNewChannel ), + bIsClient ( true ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // client connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); + + QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); +} + +CSocket::CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pServer ( pNServP ), + bIsClient ( false ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // server connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); + + QObject::connect ( this, + static_cast ( &CSocket::NewConnection ), + pServer, + &CServer::OnNewConnection ); + + QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); +} + +void CSocket::Init ( const quint16 iNewPortNumber, const quint16 iNewQosNumber, const QString& strNewServerBindIP ) { // first store parameters, in case reinit is required (mostly for iOS) - iPortNumber_ = iPortNumber; - iQosNumber_ = iQosNumber; - strServerBindIP_ = strServerBindIP; + iPortNumber = iNewPortNumber; + iQosNumber = iNewQosNumber; + strServerBindIP = strNewServerBindIP; #ifdef _WIN32 // for the Windows socket usage we have to start it up first @@ -122,41 +164,6 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const "the software is already running).", "Network Error" ); } - - // Connections ------------------------------------------------------------- - // it is important to do the following connections in this class since we - // have a thread transition - if ( bIsInitRan ) - return; - - // we have different connections for client and server - if ( bIsClient ) - { - // client connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); - - QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); - } - else - { - // server connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); - - QObject::connect ( this, - static_cast ( &CSocket::NewConnection ), - pServer, - &CServer::OnNewConnection ); - - QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); - } - - bIsInitRan = true; // QObject::connect once only } void CSocket::Close() @@ -211,7 +218,7 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr sizeof ( sockaddr_in ) ) < 0 ) { // qDebug("Socket send exception - mostly happens in iOS when returning from idle"); - Init ( iPortNumber_, iQosNumber_, strServerBindIP_ ); // reinit + Init ( iPortNumber, iQosNumber, strServerBindIP ); // reinit sendto ( UdpSocket, (const char*) &( (CVector) vecbySendBuf )[0], iVecSizeOut, diff --git a/src/socket.h b/src/socket.h index 2a248835bd..ef83e760fa 100644 --- a/src/socket.h +++ b/src/socket.h @@ -53,21 +53,9 @@ class CSocket : public QObject Q_OBJECT public: - CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : - pChannel ( pNewChannel ), - bIsClient ( true ), - bJitterBufferOK ( true ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } + CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); - CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : - pServer ( pNServP ), - bIsClient ( false ), - bJitterBufferOK ( true ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } + CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); virtual ~CSocket(); @@ -78,10 +66,9 @@ class CSocket : public QObject protected: void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); - quint16 iPortNumber_; - quint16 iQosNumber_; - QString strServerBindIP_; - bool bIsInitRan; + quint16 iPortNumber; + quint16 iQosNumber; + QString strServerBindIP; #ifdef _WIN32 SOCKET UdpSocket; From 1053b29b07d438a9e14ccaa971eb6bb83ffa34cc Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 14:49:00 +0200 Subject: [PATCH 03/14] iOS code review modified: ios/sound.h modified: src/socket.cpp --- ios/sound.h | 11 +++--- ios/sound.mm | 98 ++++++++++++++++++++++---------------------------- src/socket.cpp | 84 ++++++++++++++++++++++--------------------- src/socket.h | 23 +++--------- 4 files changed, 98 insertions(+), 118 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index aeeec03f79..b886d1c797 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -23,7 +23,6 @@ \******************************************************************************/ #pragma once -#include #include "soundbase.h" #include "global.h" @@ -56,7 +55,11 @@ class CSound : public CSoundBase bool isInitialized; protected: - MIDIPortRef midiInPortRef; - - QMutex Mutex; + void checkStatus ( int status ); + static OSStatus recordingCallback ( void* inRefCon, + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ); }; diff --git a/ios/sound.mm b/ios/sound.mm index e827f29256..9d315ef376 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -27,28 +27,39 @@ #define kOutputBus 0 #define kInputBus 1 -void checkStatus ( int status ) +/* Implementation *************************************************************/ +CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), + void* arg, + const QString& strMIDISetup, + const bool, + const QString& ) : + CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), + isInitialized ( false ) { - if ( status ) + try { - printf ( "Status not 0! %d\n", status ); - } -} + NSError* audioSessionError = nil; -void checkStatus ( int status, char* s ) -{ - if ( status ) + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; + [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { + if ( granted ) + { + // ok + } + }]; + [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; + } + catch ( const CGenErr& generr ) { - printf ( "Status not 0! %d - %s \n", status, s ); + qDebug ( "Sound exception ...." ); } } /** - This callback is called when sound card needs output data to play. And because Jamulus use the same buffer to store input and output data (input is - sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the - output fetched from server to ioData, which will then be played. + This callback is called when sound card needs output data to play. + And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. */ -static OSStatus recordingCallback ( void* inRefCon, +OSStatus CSound::recordingCallback ( void* inRefCon, AudioUnitRenderActionFlags* ioActionFlags, const AudioTimeStamp* inTimeStamp, UInt32 inBusNumber, @@ -69,14 +80,10 @@ static OSStatus recordingCallback ( void* inRefCon, bufferList.mNumberBuffers = 1; bufferList.mBuffers[0] = buffer; - // Then: // Obtain recorded samples - OSStatus status; - // Calling Unit Render to store input data to bufferList - status = AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); - // checkStatus(status, (char *)" Just called AudioUnitRender "); + AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); // Now, we have the samples we just read sitting in buffers in bufferList // Process the new data @@ -112,42 +119,12 @@ static OSStatus recordingCallback ( void* inRefCon, pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); } -/* Implementation *************************************************************/ -CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), - void* arg, - const QString& strMIDISetup, - const bool, - const QString& ) : - CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), - midiInPortRef ( static_cast ( NULL ) ) -{ - try - { - NSError* audioSessionError = nil; - - [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; - [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { - if ( granted ) - { - // ok - } - }]; - [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; - } - catch ( const CGenErr& generr ) - { - qDebug ( "Sound exception ...." ); // This try-catch seems to fix Connect button crash - } - isInitialized = false; -} - +// TODO - CSound::Init is called multiple times at launch to verify device capabilities. +// For iOS though, Init takes long, so should better reduce those inits for iOS (Now: 9 times/launch). int CSound::Init ( const int iCoreAudioBufferSizeMono ) { try { - // printf("Init sound ..."); <= to check the number of Sound inits at launch - // init base class - // CSoundBase::Init ( iCoreAudioBufferSizeMono ); this does nothing this->iCoreAudioBufferSizeMono = iCoreAudioBufferSizeMono; // set internal buffer size value and calculate stereo buffer size @@ -166,11 +143,11 @@ static OSStatus recordingCallback ( void* inRefCon, AVAudioSessionCategoryOptionAllowAirPlay error:&error]; - // NGOCDH - using values from jamulus settings 64 = 2.67ms/2 + // using values from jamulus settings 64 = 2.67ms/2 NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; - // set the session's sample rate 48000 - the only supported by Jamulus ? + // set the session's sample rate 48000 - the only supported by Jamulus [sessionInstance setPreferredSampleRate:48000 error:&error]; [[AVAudioSession sharedInstance] setActive:YES error:&error]; @@ -247,7 +224,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Init exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Init exception ...." ); } return iCoreAudioBufferSizeMono; @@ -264,7 +241,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Start exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Start exception ...." ); } } @@ -277,7 +254,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Stop exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Stop exception ...." ); } // call base class CSoundBase::Stop(); @@ -294,6 +271,7 @@ static OSStatus recordingCallback ( void* inRefCon, builtinmic = false; // try external device AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device if ( builtinmic ) { @@ -307,6 +285,14 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound dev change exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound dev change exception ...." ); + } +} + +void CSound::checkStatus ( int status ) +{ + if ( status ) + { + printf ( "Status not 0! %d\n", status ); } } diff --git a/src/socket.cpp b/src/socket.cpp index d8d41645f2..67cd38c54b 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -26,12 +26,51 @@ #include "server.h" /* Implementation *************************************************************/ -void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) + +// Connections ------------------------------------------------------------- +// it is important to do the following connections in this class since we +// have a thread transition + +// we have different connections for client and server, created after Init in corresponding constructor + +CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pChannel ( pNewChannel ), + bIsClient ( true ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // client connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); + + QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); +} + +CSocket::CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pServer ( pNServP ), + bIsClient ( false ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // server connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); + + QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pServer, &CServer::OnNewConnection ); + + QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); +} + +void CSocket::Init ( const quint16 iNewPortNumber, const quint16 iNewQosNumber, const QString& strNewServerBindIP ) { // first store parameters, in case reinit is required (mostly for iOS) - iPortNumber_ = iPortNumber; - iQosNumber_ = iQosNumber; - strServerBindIP_ = strServerBindIP; + iPortNumber = iNewPortNumber; + iQosNumber = iNewQosNumber; + strServerBindIP = strNewServerBindIP; #ifdef _WIN32 // for the Windows socket usage we have to start it up first @@ -122,41 +161,6 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const "the software is already running).", "Network Error" ); } - - // Connections ------------------------------------------------------------- - // it is important to do the following connections in this class since we - // have a thread transition - if ( bIsInitRan ) - return; - - // we have different connections for client and server - if ( bIsClient ) - { - // client connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); - - QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); - } - else - { - // server connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); - - QObject::connect ( this, - static_cast ( &CSocket::NewConnection ), - pServer, - &CServer::OnNewConnection ); - - QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); - } - - bIsInitRan = true; // QObject::connect once only } void CSocket::Close() @@ -211,7 +215,7 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr sizeof ( sockaddr_in ) ) < 0 ) { // qDebug("Socket send exception - mostly happens in iOS when returning from idle"); - Init ( iPortNumber_, iQosNumber_, strServerBindIP_ ); // reinit + Init ( iPortNumber, iQosNumber, strServerBindIP ); // reinit sendto ( UdpSocket, (const char*) &( (CVector) vecbySendBuf )[0], iVecSizeOut, diff --git a/src/socket.h b/src/socket.h index 2a248835bd..ef83e760fa 100644 --- a/src/socket.h +++ b/src/socket.h @@ -53,21 +53,9 @@ class CSocket : public QObject Q_OBJECT public: - CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : - pChannel ( pNewChannel ), - bIsClient ( true ), - bJitterBufferOK ( true ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } + CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); - CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : - pServer ( pNServP ), - bIsClient ( false ), - bJitterBufferOK ( true ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } + CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); virtual ~CSocket(); @@ -78,10 +66,9 @@ class CSocket : public QObject protected: void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); - quint16 iPortNumber_; - quint16 iQosNumber_; - QString strServerBindIP_; - bool bIsInitRan; + quint16 iPortNumber; + quint16 iQosNumber; + QString strServerBindIP; #ifdef _WIN32 SOCKET UdpSocket; From af76b99ed07225b22f95b3052590b245cafb5acb Mon Sep 17 00:00:00 2001 From: ngocdh Date: Wed, 16 Jun 2021 09:15:32 +0200 Subject: [PATCH 04/14] iOS sound - no more malloc during sound processing modified: ios/sound.mm --- ios/sound.h | 3 +++ ios/sound.mm | 50 +++++++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index b886d1c797..15c15e3b36 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -38,6 +38,7 @@ class CSound : public CSoundBase const QString& strMIDISetup, const bool, const QString& ); + ~CSound(); virtual int Init ( const int iNewPrefMonoBufferSize ); virtual void Start(); @@ -55,6 +56,8 @@ class CSound : public CSoundBase bool isInitialized; protected: + AudioBuffer buffer; + AudioBufferList bufferList; void checkStatus ( int status ); static OSStatus recordingCallback ( void* inRefCon, AudioUnitRenderActionFlags* ioActionFlags, diff --git a/ios/sound.mm b/ios/sound.mm index 9d315ef376..49ffbf7087 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -46,13 +46,27 @@ { // ok } + else + { + // TODO - alert user + } }]; [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } catch ( const CGenErr& generr ) { - qDebug ( "Sound exception ...." ); + QMessageBox::warning( nullptr, "Sound exception", generr.GetErrorText() ); } + + buffer.mNumberChannels = 2; + buffer.mData = malloc ( 256 * sizeof ( Float32 ) * 2 ); // max size + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0] = buffer; +} + +CSound::~CSound () +{ + free ( buffer.mData ); } /** @@ -69,28 +83,18 @@ And because Jamulus uses the same buffer to store input and output data (input i CSound* pSound = static_cast ( inRefCon ); - AudioBuffer buffer; - - buffer.mNumberChannels = 2; - buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * buffer.mNumberChannels; - buffer.mData = malloc ( buffer.mDataByteSize ); - - // Put buffer in a AudioBufferList - AudioBufferList bufferList; - bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0] = buffer; + // setting up temp buffer + pSound->buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * pSound->buffer.mNumberChannels; + pSound->bufferList.mBuffers[0] = pSound->buffer; // Obtain recorded samples // Calling Unit Render to store input data to bufferList - AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); + AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &pSound->bufferList ); // Now, we have the samples we just read sitting in buffers in bufferList // Process the new data - pSound->processBufferList ( &bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList - - // release the malloc'ed data in the buffer we created earlier - free ( bufferList.mBuffers[0].mData ); + pSound->processBufferList ( &pSound->bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList Float32* pData = (Float32*) ( ioData->mBuffers[0].mData ); @@ -144,11 +148,11 @@ And because Jamulus uses the same buffer to store input and output data (input i error:&error]; // using values from jamulus settings 64 = 2.67ms/2 - NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math + NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / Float32 ( SYSTEM_SAMPLE_RATE_HZ ); // yeah it's math [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; // set the session's sample rate 48000 - the only supported by Jamulus - [sessionInstance setPreferredSampleRate:48000 error:&error]; + [sessionInstance setPreferredSampleRate:SYSTEM_SAMPLE_RATE_HZ error:&error]; [[AVAudioSession sharedInstance] setActive:YES error:&error]; OSStatus status; @@ -179,7 +183,7 @@ And because Jamulus uses the same buffer to store input and output data (input i // Describe format AudioStreamBasicDescription audioFormat; - audioFormat.mSampleRate = 48000.00; + audioFormat.mSampleRate = SYSTEM_SAMPLE_RATE_HZ; audioFormat.mFormatID = kAudioFormatLinearPCM; audioFormat.mFormatFlags = kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; audioFormat.mFramesPerPacket = 1; @@ -224,7 +228,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Init exception ...." ); + QMessageBox::warning( nullptr, "Sound init exception", generr.GetErrorText() ); } return iCoreAudioBufferSizeMono; @@ -241,7 +245,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Start exception ...." ); + QMessageBox::warning( nullptr, "Sound start exception", generr.GetErrorText() ); } } @@ -254,7 +258,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Stop exception ...." ); + QMessageBox::warning( nullptr, "Sound stop exception", generr.GetErrorText() ); } // call base class CSoundBase::Stop(); @@ -285,7 +289,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound dev change exception ...." ); + QMessageBox::warning( nullptr, "Sound device change exception", generr.GetErrorText() ); } } From 40221f2403df4e82f70c9d1672b853829bf5b18f Mon Sep 17 00:00:00 2001 From: ngocdh Date: Sat, 19 Jun 2021 09:31:20 +0200 Subject: [PATCH 05/14] Remove iOS device switch feature to keep code consistent --- ios/sound.h | 1 - ios/sound.mm | 29 ----------------------------- src/client.cpp | 9 --------- src/client.h | 6 ------ src/clientdlg.cpp | 11 ----------- src/clientdlg.h | 14 -------------- src/settings.h | 2 -- 7 files changed, 72 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index 15c15e3b36..904b6a2ad2 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -44,7 +44,6 @@ class CSound : public CSoundBase virtual void Start(); virtual void Stop(); virtual void processBufferList ( AudioBufferList*, CSound* ); - virtual void SetInputDeviceId ( int deviceid ); AudioUnit audioUnit; diff --git a/ios/sound.mm b/ios/sound.mm index 49ffbf7087..a04e57a145 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -264,35 +264,6 @@ And because Jamulus uses the same buffer to store input and output data (input i CSoundBase::Stop(); } -void CSound::SetInputDeviceId ( int deviceid ) -{ - try - { - NSError* error = nil; - bool builtinmic = true; - - if ( deviceid == 0 ) - builtinmic = false; // try external device - - AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; - - // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device - if ( builtinmic ) - { - [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; - } - else - { - unsigned long lastInput = sessionInstance.availableInputs.count - 1; - [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; - } - } - catch ( const CGenErr& generr ) - { - QMessageBox::warning( nullptr, "Sound device change exception", generr.GetErrorText() ); - } -} - void CSound::checkStatus ( int status ) { if ( status ) diff --git a/src/client.cpp b/src/client.cpp index 41129e132a..dc777de4e9 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -52,7 +52,6 @@ CClient::CClient ( const quint16 iPortNumber, bReverbOnLeftChan ( false ), iReverbLevel ( 0 ), iInputBoost ( 1 ), - iBuiltInMicId ( 0 ), iSndCrdPrefFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), iSndCrdFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), bSndCrdConversionBufferRequired ( false ), @@ -1251,11 +1250,3 @@ int CClient::EstimatedOverallDelay ( const int iPingTimeMs ) return MathUtils::round ( fTotalBufferDelayMs + iPingTimeMs ); } - -void CClient::SetInputDeviceId ( const int deviceid ) -{ -#if defined( Q_OS_IOS ) - // iOS only - Sound.SetInputDeviceId ( deviceid ); -#endif -} diff --git a/src/client.h b/src/client.h index dc5d816920..929da28853 100644 --- a/src/client.h +++ b/src/client.h @@ -240,9 +240,6 @@ class CClient : public QObject void SetInputBoost ( const int iNewBoost ) { iInputBoost = iNewBoost; } - void SetBuiltInMicId ( const int iNewMicId ) { iBuiltInMicId = iNewMicId; } - int GetBuiltInMicId() { return iBuiltInMicId; } - void SetRemoteInfo() { Channel.SetRemoteInfo ( ChannelInfo ); } void CreateChatTextMes ( const QString& strChatText ) { Channel.CreateChatTextMes ( strChatText ); } @@ -267,8 +264,6 @@ class CClient : public QObject Channel.GetBufErrorRates ( vecErrRates, dLimit, dMaxUpLimit ); } - void SetInputDeviceId ( const int deviceid ); // for mobile devices - 0 for external devices - // settings CChannelCoreInfo ChannelInfo; QString strClientName; @@ -329,7 +324,6 @@ class CClient : public QObject int iReverbLevel; CAudioReverb AudioReverb; int iInputBoost; - int iBuiltInMicId; int iSndCrdPrefFrameSizeFactor; int iSndCrdFrameSizeFactor; diff --git a/src/clientdlg.cpp b/src/clientdlg.cpp index d4634fc162..b34d9e741c 100644 --- a/src/clientdlg.cpp +++ b/src/clientdlg.cpp @@ -375,17 +375,6 @@ CClientDlg::CClientDlg ( CClient* pNCliP, pMenu->addMenu ( pEditMenu ); pMenu->addMenu ( new CHelpMenu ( true, this ) ); -#if defined( Q_OS_IOS ) - // Mobile input -------------------------------------------------------------- - QMenu* pMobileMenu = new QMenu ( tr ( "&Mobile input" ), this ); - - pMobileMenu->addAction ( tr ( "Builtin Mic" ), this, SLOT ( setBuiltinMic() ) ); - - pMobileMenu->addAction ( tr ( "Auto" ), this, SLOT ( unsetBuiltinMic() ) ); - - pMenu->addMenu ( pMobileMenu ); -#endif - // Now tell the layout about the menu layout()->setMenuBar ( pMenu ); diff --git a/src/clientdlg.h b/src/clientdlg.h index dcfb49ddcd..4d2ce7e182 100644 --- a/src/clientdlg.h +++ b/src/clientdlg.h @@ -231,20 +231,6 @@ public slots: void accept() { close(); } // introduced by pljones - void setBuiltinMic() - { -#if defined( Q_OS_IOS ) - pClient->SetInputDeviceId ( 1 ); -#endif - } - - void unsetBuiltinMic() - { -#if defined( Q_OS_IOS ) - pClient->SetInputDeviceId ( 0 ); -#endif - } - signals: void SendTabChange ( int iTabIdx ); }; diff --git a/src/settings.h b/src/settings.h index 57e28b76fc..765b6ee115 100644 --- a/src/settings.h +++ b/src/settings.h @@ -120,7 +120,6 @@ class CClientSettings : public CSettings vstrIPAddress ( MAX_NUM_SERVER_ADDR_ITEMS, "" ), iNewClientFaderLevel ( 100 ), iInputBoost ( 1 ), - iBuiltInMicId ( 0 ), iSettingsTab ( SETTING_TAB_AUDIONET ), bConnectDlgShowAllMusicians ( true ), eChannelSortType ( ST_NO_SORT ), @@ -153,7 +152,6 @@ class CClientSettings : public CSettings int iNewClientFaderLevel; int iInputBoost; int iSettingsTab; - int iBuiltInMicId; // 0 for external bool bConnectDlgShowAllMusicians; EChSortType eChannelSortType; int iNumMixerPanelRows; From be2d552ddd78e35167386bce28e5198c6dc428f9 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Tue, 29 Jun 2021 10:52:04 +0200 Subject: [PATCH 06/14] iOS switch device feature modified: ios/sound.mm --- ios/sound.h | 6 +++ ios/sound.mm | 121 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index 904b6a2ad2..f6bdee07f3 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -55,6 +55,10 @@ class CSound : public CSoundBase bool isInitialized; protected: + virtual QString LoadAndInitializeDriver ( QString strDriverName, bool ); + void GetAvailableInOutDevices(); + void SwitchDevice ( QString strDriverName ); + AudioBuffer buffer; AudioBufferList bufferList; void checkStatus ( int status ); @@ -64,4 +68,6 @@ class CSound : public CSoundBase UInt32 inBusNumber, UInt32 inNumberFrames, AudioBufferList* ioData ); + + QMutex Mutex; }; diff --git a/ios/sound.mm b/ios/sound.mm index a04e57a145..eba38b8a11 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -42,50 +42,48 @@ [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { - if ( granted ) - { - // ok - } - else - { - // TODO - alert user - } + if ( granted ) + { + // ok + } + else + { + // TODO - alert user + } }]; [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound exception", generr.GetErrorText() ); } buffer.mNumberChannels = 2; - buffer.mData = malloc ( 256 * sizeof ( Float32 ) * 2 ); // max size + buffer.mData = malloc ( 256 * sizeof ( Float32 ) * buffer.mNumberChannels ); // max size bufferList.mNumberBuffers = 1; bufferList.mBuffers[0] = buffer; } -CSound::~CSound () -{ - free ( buffer.mData ); -} +CSound::~CSound() { free ( buffer.mData ); } /** This callback is called when sound card needs output data to play. - And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. + And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually + use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. */ OSStatus CSound::recordingCallback ( void* inRefCon, - AudioUnitRenderActionFlags* ioActionFlags, - const AudioTimeStamp* inTimeStamp, - UInt32 inBusNumber, - UInt32 inNumberFrames, - AudioBufferList* ioData ) + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ) { CSound* pSound = static_cast ( inRefCon ); // setting up temp buffer pSound->buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * pSound->buffer.mNumberChannels; - pSound->bufferList.mBuffers[0] = pSound->buffer; + pSound->bufferList.mBuffers[0] = pSound->buffer; // Obtain recorded samples @@ -224,11 +222,29 @@ And because Jamulus uses the same buffer to store input and output data (input i status = AudioUnitInitialize ( audioUnit ); checkStatus ( status ); + SwitchDevice ( strCurDevName ); + + if ( !isInitialized ) + { + [[NSNotificationCenter defaultCenter] addObserverForName:AVAudioSessionRouteChangeNotification + object:nil + queue:nil + usingBlock:^( NSNotification* notification ) { + UInt8 reason = + [[notification.userInfo valueForKey:AVAudioSessionRouteChangeReasonKey] intValue]; + if ( reason == AVAudioSessionRouteChangeReasonNewDeviceAvailable or + reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable ) + { + emit ReinitRequest ( RS_RELOAD_RESTART_AND_INIT ); // reload the available devices frame + } + }]; + } + isInitialized = true; } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound init exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound init exception", generr.GetErrorText() ); } return iCoreAudioBufferSizeMono; @@ -245,7 +261,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound start exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound start exception", generr.GetErrorText() ); } } @@ -258,7 +274,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound stop exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound stop exception", generr.GetErrorText() ); } // call base class CSoundBase::Stop(); @@ -271,3 +287,60 @@ And because Jamulus uses the same buffer to store input and output data (input i printf ( "Status not 0! %d\n", status ); } } + +QString CSound::LoadAndInitializeDriver ( QString strDriverName, bool ) +{ + // secure lNumDevs/strDriverNames access + QMutexLocker locker ( &Mutex ); + + // reload the driver list of available sound devices + GetAvailableInOutDevices(); + + // store the current name of the driver + strCurDevName = strDriverName; + + return ""; +} + +void CSound::GetAvailableInOutDevices() +{ + // always add system default devices for input and output as first entry + lNumDevs = 1; + strDriverNames[0] = "System Default In/Out Devices"; + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + if ( sessionInstance.availableInputs.count > 1 ) + { + lNumDevs = 2; + strDriverNames[1] = "in: Built-in Mic/out: System Default"; + } +} + +void CSound::SwitchDevice ( QString strDriverName ) +{ + // find driver index from given driver name + int iDriverIdx = INVALID_INDEX; // initialize with an invalid index + + for ( int i = 0; i < MAX_NUMBER_SOUND_CARDS; i++ ) + { + if ( strDriverName.compare ( strDriverNames[i] ) == 0 ) + { + iDriverIdx = i; + } + } + + NSError* error = nil; + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + if ( iDriverIdx == 0 ) // system default device + { + unsigned long lastInput = sessionInstance.availableInputs.count - 1; + [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; + } + else // built-in mic + { + [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; + } +} From 2f21485d15e82064003bb7f042853bb40d83b60d Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 09:56:32 +0200 Subject: [PATCH 07/14] iOS sound + feature device switch for iOS --- Jamulus.pro | 8 +- ios/Info.plist | 4 + ios/ios_app_delegate.mm | 7 +- ios/sound.h | 61 +----- ios/sound.mm | 404 +++++++++++++++++++++++--------------- src/client.cpp | 23 +++ src/client.h | 6 + src/clientdlg.cpp | 11 ++ src/clientdlg.h | 14 ++ src/clientsettingsdlg.cpp | 2 +- src/main.cpp | 1 + src/settings.h | 2 + src/socket.cpp | 121 +++++++----- src/socket.h | 6 +- 14 files changed, 398 insertions(+), 272 deletions(-) diff --git a/Jamulus.pro b/Jamulus.pro index 9778b3daf2..89072d5bd6 100644 --- a/Jamulus.pro +++ b/Jamulus.pro @@ -196,12 +196,8 @@ win32 { OBJECTIVE_SOURCES += ios/sound.mm QMAKE_TARGET_BUNDLE_PREFIX = io.jamulus QMAKE_APPLICATION_BUNDLE_NAME. = $$TARGET - LIBS += -framework CoreFoundation \ - -framework CoreServices \ - -framework AVFoundation \ - -framework CoreMIDI \ - -framework AudioToolbox \ - -framework Foundation + LIBS += -framework AVFoundation \ + -framework AudioToolbox } else:android { # we want to compile with C++14 CONFIG += c++14 diff --git a/ios/Info.plist b/ios/Info.plist index dfc2d2937c..aa4a9559b8 100644 --- a/ios/Info.plist +++ b/ios/Info.plist @@ -29,6 +29,10 @@ UIApplicationSupportsIndirectInputEvents + UIBackgroundModes + + audio + UILaunchScreen UILaunchStoryboardName diff --git a/ios/ios_app_delegate.mm b/ios/ios_app_delegate.mm index 54b78a2651..42a1dbee3b 100644 --- a/ios/ios_app_delegate.mm +++ b/ios/ios_app_delegate.mm @@ -1,4 +1,3 @@ - #import "ios_app_delegate.h" @interface QIOSApplicationDelegate () @@ -7,10 +6,8 @@ @interface QIOSApplicationDelegate () @implementation QIOSApplicationDelegate - - -- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions { - +- (BOOL)application:(UIApplication*)application didFinishLaunchingWithOptions:(NSDictionary*)launchOptions +{ return YES; } @end diff --git a/ios/sound.h b/ios/sound.h index 7f08179867..aeeec03f79 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -1,8 +1,8 @@ /******************************************************************************\ - * Copyright (c) 2004-2020 + * Copyright (c) 2004-2021 * * Author(s): - * ann0see based on code from Volker Fischer + * ann0see and ngocdh based on code from Volker Fischer * ****************************************************************************** * @@ -23,13 +23,12 @@ \******************************************************************************/ #pragma once -#include #include -#include #include "soundbase.h" #include "global.h" -/* Classes ********************************************************************/ +#import + class CSound : public CSoundBase { Q_OBJECT @@ -44,64 +43,20 @@ class CSound : public CSoundBase virtual int Init ( const int iNewPrefMonoBufferSize ); virtual void Start(); virtual void Stop(); + virtual void processBufferList ( AudioBufferList*, CSound* ); + virtual void SetInputDeviceId ( int deviceid ); - // channel selection - virtual int GetNumInputChannels() { return iNumInChanPlusAddChan; } - virtual QString GetInputChannelName ( const int iDiD ) { return sChannelNamesInput[iDiD]; } - virtual void SetLeftInputChannel ( const int iNewChan ); - virtual void SetRightInputChannel ( const int iNewChan ); - virtual int GetLeftInputChannel() { return iSelInputLeftChannel; } - virtual int GetRightInputChannel() { return iSelInputRightChannel; } - - virtual int GetNumOutputChannels() { return iNumOutChan; } - virtual QString GetOutputChannelName ( const int iDiD ) { return sChannelNamesOutput[iDiD]; } - virtual void SetLeftOutputChannel ( const int iNewChan ); - virtual void SetRightOutputChannel ( const int iNewChan ); - virtual int GetLeftOutputChannel() { return iSelOutputLeftChannel; } - virtual int GetRightOutputChannel() { return iSelOutputRightChannel; } + AudioUnit audioUnit; // these variables/functions should be protected but cannot since we want // to access them from the callback function CVector vecsTmpAudioSndCrdStereo; int iCoreAudioBufferSizeMono; int iCoreAudioBufferSizeStereo; - long lCurDev; - int iNumInChan; - int iNumInChanPlusAddChan; // includes additional "added" channels - int iNumOutChan; - int iSelInputLeftChannel; - int iSelInputRightChannel; - int iSelOutputLeftChannel; - int iSelOutputRightChannel; - int iSelInBufferLeft; - int iSelInBufferRight; - int iSelInInterlChLeft; - int iSelInInterlChRight; - int iSelAddInBufferLeft; - int iSelAddInBufferRight; - int iSelAddInInterlChLeft; - int iSelAddInInterlChRight; - int iSelOutBufferLeft; - int iSelOutBufferRight; - int iSelOutInterlChLeft; - int iSelOutInterlChRight; - CVector vecNumInBufChan; - CVector vecNumOutBufChan; + bool isInitialized; protected: - virtual QString LoadAndInitializeDriver ( QString strDriverName, bool ); - - QString CheckDeviceCapabilities ( const int iDriverIdx ); - void GetAvailableInOutDevices(); - - static void callbackMIDI ( const MIDIPacketList* pktlist, void* refCon, void* ); - - // AVAudioSession audioSession; - MIDIPortRef midiInPortRef; - QString sChannelNamesInput[MAX_NUM_IN_OUT_CHANNELS]; - QString sChannelNamesOutput[MAX_NUM_IN_OUT_CHANNELS]; - QMutex Mutex; }; diff --git a/ios/sound.mm b/ios/sound.mm index a768b14876..e827f29256 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -1,8 +1,8 @@ /******************************************************************************\ - * Copyright (c) 2004-2020 + * Copyright (c) 2004-2021 * * Author(s): - * ann0see based on code from Volker Fischer + * ann0see and ngocdh based on code from Volker Fischer * ****************************************************************************** * @@ -24,209 +24,289 @@ #include "sound.h" #include +#define kOutputBus 0 +#define kInputBus 1 -/* Implementation *************************************************************/ -CSound::CSound ( void (*fpNewProcessCallback) ( CVector& psData, void* arg ), - void* arg, - const QString& strMIDISetup, - const bool , - const QString& ) : - CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), - midiInPortRef ( static_cast ( NULL ) ) +void checkStatus ( int status ) { - NSError *audioSessionError = nil; - - [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; - [[AVAudioSession sharedInstance] requestRecordPermission:^(BOOL granted) { - if (granted) { - // ok - } - }]; - [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; + if ( status ) + { + printf ( "Status not 0! %d\n", status ); + } +} - //GetAvailableInOutDevices(); +void checkStatus ( int status, char* s ) +{ + if ( status ) + { + printf ( "Status not 0! %d - %s \n", status, s ); + } } -int CSound::Init ( const int /*iNewPrefMonoBufferSize*/ ) +/** + This callback is called when sound card needs output data to play. And because Jamulus use the same buffer to store input and output data (input is + sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the + output fetched from server to ioData, which will then be played. + */ +static OSStatus recordingCallback ( void* inRefCon, + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ) { - // store buffer size - //iCoreAudioBufferSizeMono = audioSession.IOBufferDuration; + CSound* pSound = static_cast ( inRefCon ); - // init base class - //CSoundBase::Init ( iCoreAudioBufferSizeMono ); + AudioBuffer buffer; - // set internal buffer size value and calculate stereo buffer size - //iCoreAudioBufferSizeStereo = 2 * iCoreAudioBufferSizeMono; + buffer.mNumberChannels = 2; + buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * buffer.mNumberChannels; + buffer.mData = malloc ( buffer.mDataByteSize ); - // create memory for intermediate audio buffer - //vecsTmpAudioSndCrdStereo.Init ( iCoreAudioBufferSizeStereo ); + // Put buffer in a AudioBufferList + AudioBufferList bufferList; + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0] = buffer; - return iCoreAudioBufferSizeMono; -} + // Then: + // Obtain recorded samples -void CSound::Start() -{ - // ?? + OSStatus status; - // call base class - //CSoundBase::Start(); -} + // Calling Unit Render to store input data to bufferList + status = AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); + // checkStatus(status, (char *)" Just called AudioUnitRender "); -void CSound::Stop() -{ - // ?? - // call base class - //CSoundBase::Stop(); -} + // Now, we have the samples we just read sitting in buffers in bufferList + // Process the new data + pSound->processBufferList ( &bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList -void CSound::GetAvailableInOutDevices() -{ - // https://developer.apple.com/documentation/avfoundation/avaudiosession/1616557-availableinputs?language=objc? + // release the malloc'ed data in the buffer we created earlier + free ( bufferList.mBuffers[0].mData ); + + Float32* pData = (Float32*) ( ioData->mBuffers[0].mData ); + + // copy output data + for ( int i = 0; i < pSound->iCoreAudioBufferSizeMono; i++ ) + { + pData[2 * i] = (Float32) pSound->vecsTmpAudioSndCrdStereo[2 * i] / _MAXSHORT; // left + pData[2 * i + 1] = (Float32) pSound->vecsTmpAudioSndCrdStereo[2 * i + 1] / _MAXSHORT; // right + } + return noErr; } -QString CSound::LoadAndInitializeDriver ( QString strDriverName, bool ) +void CSound::processBufferList ( AudioBufferList* inInputData, CSound* pSound ) // got stereo input data { - // get the driver: check if devices are capable - // reload the driver list of available sound devices - //GetAvailableInOutDevices(); - - // find driver index from given driver name - //int iDriverIdx = INVALID_INDEX; // initialize with an invalid index - - /*for ( int i = 0; i < MAX_NUMBER_SOUND_CARDS; i++ ) - { - if ( strDriverName.compare ( strDriverNames[i] ) == 0 ) - { - iDriverIdx = i; - } - } - - // if the selected driver was not found, return an error message - if ( iDriverIdx == INVALID_INDEX ) - { - return tr ( "The current selected audio device is no longer present in the system." ); - } - - // check device capabilities if it fulfills our requirements - const QString strStat = CheckDeviceCapabilities ( iDriverIdx ); - - // check if device is capable and if not the same device is used - if ( strStat.isEmpty() && ( strCurDevName.compare ( strDriverNames[iDriverIdx] ) != 0 ) ) - { - } - // set the left/right in/output channels - - return strStat; - */ - return ""; + QMutexLocker locker ( &pSound->MutexAudioProcessCallback ); + Float32* pData = static_cast ( inInputData->mBuffers[0].mData ); + + // copy input data + for ( int i = 0; i < pSound->iCoreAudioBufferSizeMono; i++ ) + { + // copy left and right channels separately + pSound->vecsTmpAudioSndCrdStereo[2 * i] = (short) ( pData[2 * i] * _MAXSHORT ); // left + pSound->vecsTmpAudioSndCrdStereo[2 * i + 1] = (short) ( pData[2 * i + 1] * _MAXSHORT ); // right + } + pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); } -QString CSound::CheckDeviceCapabilities ( const int iDriverIdx ) +/* Implementation *************************************************************/ +CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), + void* arg, + const QString& strMIDISetup, + const bool, + const QString& ) : + CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), + midiInPortRef ( static_cast ( NULL ) ) { - /*// This function checks if our required input/output channel - // properties are supported by the selected device. If the return - // string is empty, the device can be used, otherwise the error - // message is returned. - - // check sample rate of input - audioSession.setPreferredSampleRate( SYSTEM_SAMPLE_RATE_HZ ); - // if false, try to set it - if ( audioSession.sampleRate != SYSTEM_SAMPLE_RATE_HZ ) - { - throw CGenErr ( tr ("Could not set sample rate. Please close other applications playing audio.") ); - } - - // special case with 4 input channels: support adding channels - if ( ( lNumInChan == 4 ) && bInputChMixingSupported ) + try { - // add four mixed channels (i.e. 4 normal, 4 mixed channels) - lNumInChanPlusAddChan = 8; - - for ( int iCh = 0; iCh < lNumInChanPlusAddChan; iCh++ ) - { - int iSelCH, iSelAddCH; - - GetSelCHAndAddCH ( iCh, lNumInChan, iSelCH, iSelAddCH ); - - if ( iSelAddCH >= 0 ) - { - // for mixed channels, show both audio channel names to be mixed - channelInputName[iCh] = - channelInputName[iSelCH] + " + " + channelInputName[iSelAddCH]; - } - } + NSError* audioSessionError = nil; + + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; + [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { + if ( granted ) + { + // ok + } + }]; + [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } - else + catch ( const CGenErr& generr ) { - // regular case: no mixing input channels used - lNumInChanPlusAddChan = lNumInChan; + qDebug ( "Sound exception ...." ); // This try-catch seems to fix Connect button crash } -*/ - // everything is ok, return empty string for "no error" case - return ""; + isInitialized = false; } -void CSound::SetLeftInputChannel ( const int iNewChan ) +int CSound::Init ( const int iCoreAudioBufferSizeMono ) { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < iNumInChanPlusAddChan ) ) + try { - iSelInputLeftChannel = iNewChan; + // printf("Init sound ..."); <= to check the number of Sound inits at launch + // init base class + // CSoundBase::Init ( iCoreAudioBufferSizeMono ); this does nothing + this->iCoreAudioBufferSizeMono = iCoreAudioBufferSizeMono; + + // set internal buffer size value and calculate stereo buffer size + iCoreAudioBufferSizeStereo = 2 * iCoreAudioBufferSizeMono; + + // create memory for intermediate audio buffer + vecsTmpAudioSndCrdStereo.Init ( iCoreAudioBufferSizeStereo ); + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + // we are going to play and record so we pick that category + NSError* error = nil; + [sessionInstance setCategory:AVAudioSessionCategoryPlayAndRecord + withOptions:AVAudioSessionCategoryOptionMixWithOthers | AVAudioSessionCategoryOptionDefaultToSpeaker | + AVAudioSessionCategoryOptionAllowBluetooth | AVAudioSessionCategoryOptionAllowBluetoothA2DP | + AVAudioSessionCategoryOptionAllowAirPlay + error:&error]; + + // NGOCDH - using values from jamulus settings 64 = 2.67ms/2 + NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math + [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; + + // set the session's sample rate 48000 - the only supported by Jamulus ? + [sessionInstance setPreferredSampleRate:48000 error:&error]; + [[AVAudioSession sharedInstance] setActive:YES error:&error]; + + OSStatus status; + + // Describe audio component + AudioComponentDescription desc; + desc.componentType = kAudioUnitType_Output; + desc.componentSubType = kAudioUnitSubType_RemoteIO; + desc.componentFlags = 0; + desc.componentFlagsMask = 0; + desc.componentManufacturer = kAudioUnitManufacturer_Apple; + + // Get component + AudioComponent inputComponent = AudioComponentFindNext ( NULL, &desc ); + + // Get audio units + status = AudioComponentInstanceNew ( inputComponent, &audioUnit ); + checkStatus ( status ); + + // Enable IO for recording + UInt32 flag = 1; + status = AudioUnitSetProperty ( audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, kInputBus, &flag, sizeof ( flag ) ); + checkStatus ( status ); + + // Enable IO for playback + status = AudioUnitSetProperty ( audioUnit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, kOutputBus, &flag, sizeof ( flag ) ); + checkStatus ( status ); + + // Describe format + AudioStreamBasicDescription audioFormat; + audioFormat.mSampleRate = 48000.00; + audioFormat.mFormatID = kAudioFormatLinearPCM; + audioFormat.mFormatFlags = kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; + audioFormat.mFramesPerPacket = 1; + audioFormat.mChannelsPerFrame = 2; // steoreo, so 2 interleaved channels + audioFormat.mBitsPerChannel = 32; // sizeof float32 + audioFormat.mBytesPerPacket = 8; // (sizeof float32) * 2 channels + audioFormat.mBytesPerFrame = 8; //(sizeof float32) * 2 channels + + // Apply format + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Output, + kInputBus, + &audioFormat, + sizeof ( audioFormat ) ); + checkStatus ( status ); + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_StreamFormat, + kAudioUnitScope_Input, + kOutputBus, + &audioFormat, + sizeof ( audioFormat ) ); + checkStatus ( status ); + + // Set callback + AURenderCallbackStruct callbackStruct; + callbackStruct.inputProc = recordingCallback; // this is actually the playback callback + callbackStruct.inputProcRefCon = this; + status = AudioUnitSetProperty ( audioUnit, + kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Global, + kOutputBus, + &callbackStruct, + sizeof ( callbackStruct ) ); + checkStatus ( status ); + + // Initialise + status = AudioUnitInitialize ( audioUnit ); + checkStatus ( status ); + + isInitialized = true; + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Init exception ...." ); // This try-catch seems to fix Connect button crash } - */ -} -void CSound::SetRightInputChannel ( const int iNewChan ) -{ - /* // apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumInChanPlusAddChan ) ) - { - vSelectedInputChannels[1] = iNewChan; - }*/ + return iCoreAudioBufferSizeMono; } -void CSound::SetLeftOutputChannel ( const int iNewChan ) +void CSound::Start() { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumOutChan ) ) - { - vSelectedOutputChannels[0] = iNewChan; - }*/ + // call base class + CSoundBase::Start(); + try + { + OSStatus err = AudioOutputUnitStart ( audioUnit ); + checkStatus ( err ); + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Start exception ...." ); // This try-catch seems to fix Connect button crash + } } -void CSound::SetRightOutputChannel ( const int iNewChan ) +void CSound::Stop() { - /*// apply parameter after input parameter check - if ( ( iNewChan >= 0 ) && ( iNewChan < lNumOutChan ) ) - { - vSelectedOutputChannels[1] = iNewChan; - }*/ + try + { + OSStatus err = AudioOutputUnitStop ( audioUnit ); + checkStatus ( err ); + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound Stop exception ...." ); // This try-catch seems to fix Connect button crash + } + // call base class + CSoundBase::Stop(); } - -void CSound::callbackMIDI ( const MIDIPacketList* pktlist, - void* refCon, - void* ) +void CSound::SetInputDeviceId ( int deviceid ) { - /* CSound* pSound = static_cast ( refCon ); - - if ( pSound->midiInPortRef != static_cast ( NULL ) ) + try { - MIDIPacket* midiPacket = const_cast ( pktlist->packet ); + NSError* error = nil; + bool builtinmic = true; - for ( unsigned int j = 0; j < pktlist->numPackets; j++ ) + if ( deviceid == 0 ) + builtinmic = false; // try external device + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device + if ( builtinmic ) + { + [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; + } + else { - // copy packet and send it to the MIDI parser - CVector vMIDIPaketBytes ( midiPacket->length ); - for ( int i = 0; i < midiPacket->length; i++ ) - { - vMIDIPaketBytes[i] = static_cast ( midiPacket->data[i] ); - } - pSound->ParseMIDIMessage ( vMIDIPaketBytes ); - - midiPacket = MIDIPacketNext ( midiPacket ); + unsigned long lastInput = sessionInstance.availableInputs.count - 1; + [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; } - }*/ + } + catch ( const CGenErr& generr ) + { + qDebug ( "Sound dev change exception ...." ); // This try-catch seems to fix Connect button crash + } } diff --git a/src/client.cpp b/src/client.cpp index 240a2e122b..20237b66fe 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -53,6 +53,7 @@ CClient::CClient ( const quint16 iPortNumber, bReverbOnLeftChan ( false ), iReverbLevel ( 0 ), iInputBoost ( 1 ), + iBuiltInMicId ( 0 ), iSndCrdPrefFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), iSndCrdFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), bSndCrdConversionBufferRequired ( false ), @@ -769,14 +770,28 @@ void CClient::Init() const int iFraSizeDefault = SYSTEM_FRAME_SIZE_SAMPLES * FRAME_SIZE_FACTOR_DEFAULT; const int iFraSizeSafe = SYSTEM_FRAME_SIZE_SAMPLES * FRAME_SIZE_FACTOR_SAFE; +#if defined( Q_OS_IOS ) + bFraSiFactPrefSupported = true; // to reduce sound init time, because we know it's supported in iOS + bFraSiFactDefSupported = true; + bFraSiFactSafeSupported = true; +#else bFraSiFactPrefSupported = ( Sound.Init ( iFraSizePreffered ) == iFraSizePreffered ); bFraSiFactDefSupported = ( Sound.Init ( iFraSizeDefault ) == iFraSizeDefault ); bFraSiFactSafeSupported = ( Sound.Init ( iFraSizeSafe ) == iFraSizeSafe ); +#endif // translate block size index in actual block size const int iPrefMonoFrameSize = iSndCrdPrefFrameSizeFactor * SYSTEM_FRAME_SIZE_SAMPLES; // get actual sound card buffer size using preferred size + // TODO - iOS needs 1 init only, now: 9 inits at launch <- slow + // Initially, I tried to fix this as follows (inside #ifdef ios tag): + // if ( Sound.isInitialized ) + // iMonoBlockSizeSam = iPrefMonoFrameSize; + // else + // iMonoBlockSizeSam = Sound.Init ( iPrefMonoFrameSize ); + // Problem is legitimate setting changes (buffer size for example). + // so the condition should be something like "if ( Sound.isInitialized and APP_IS_INIALIZING)" iMonoBlockSizeSam = Sound.Init ( iPrefMonoFrameSize ); // Calculate the current sound card frame size factor. In case @@ -1238,3 +1253,11 @@ int CClient::EstimatedOverallDelay ( const int iPingTimeMs ) return MathUtils::round ( fTotalBufferDelayMs + iPingTimeMs ); } + +void CClient::SetInputDeviceId ( const int deviceid ) +{ +#if defined( Q_OS_IOS ) + // iOS only + Sound.SetInputDeviceId ( deviceid ); +#endif +} diff --git a/src/client.h b/src/client.h index ca82328080..a7b731d45a 100644 --- a/src/client.h +++ b/src/client.h @@ -241,6 +241,9 @@ class CClient : public QObject void SetInputBoost ( const int iNewBoost ) { iInputBoost = iNewBoost; } + void SetBuiltInMicId ( const int iNewMicId ) { iBuiltInMicId = iNewMicId; } + int GetBuiltInMicId() { return iBuiltInMicId; } + void SetRemoteInfo() { Channel.SetRemoteInfo ( ChannelInfo ); } void CreateChatTextMes ( const QString& strChatText ) { Channel.CreateChatTextMes ( strChatText ); } @@ -265,6 +268,8 @@ class CClient : public QObject Channel.GetBufErrorRates ( vecErrRates, dLimit, dMaxUpLimit ); } + void SetInputDeviceId ( const int deviceid ); // for mobile devices - 0 for external devices + // settings CChannelCoreInfo ChannelInfo; QString strClientName; @@ -325,6 +330,7 @@ class CClient : public QObject int iReverbLevel; CAudioReverb AudioReverb; int iInputBoost; + int iBuiltInMicId; int iSndCrdPrefFrameSizeFactor; int iSndCrdFrameSizeFactor; diff --git a/src/clientdlg.cpp b/src/clientdlg.cpp index 9de0b354f0..14034bc2cc 100644 --- a/src/clientdlg.cpp +++ b/src/clientdlg.cpp @@ -386,6 +386,17 @@ CClientDlg::CClientDlg ( CClient* pNCliP, pMenu->addMenu ( pSettingsMenu ); pMenu->addMenu ( new CHelpMenu ( true, this ) ); +#if defined( Q_OS_IOS ) + // Mobile input -------------------------------------------------------------- + QMenu* pMobileMenu = new QMenu ( tr ( "&Mobile input" ), this ); + + pMobileMenu->addAction ( tr ( "Builtin Mic" ), this, SLOT ( setBuiltinMic() ) ); + + pMobileMenu->addAction ( tr ( "Auto" ), this, SLOT ( unsetBuiltinMic() ) ); + + pMenu->addMenu ( pMobileMenu ); +#endif + // Now tell the layout about the menu layout()->setMenuBar ( pMenu ); diff --git a/src/clientdlg.h b/src/clientdlg.h index 5437fb8b2a..6a3cef6e46 100644 --- a/src/clientdlg.h +++ b/src/clientdlg.h @@ -233,6 +233,20 @@ public slots: void accept() { close(); } // introduced by pljones + void setBuiltinMic() + { +#if defined( Q_OS_IOS ) + pClient->SetInputDeviceId ( 1 ); +#endif + } + + void unsetBuiltinMic() + { +#if defined( Q_OS_IOS ) + pClient->SetInputDeviceId ( 0 ); +#endif + } + signals: void SendTabChange ( int iTabIdx ); }; diff --git a/src/clientsettingsdlg.cpp b/src/clientsettingsdlg.cpp index f0bea55c83..ebf8fb3755 100644 --- a/src/clientsettingsdlg.cpp +++ b/src/clientsettingsdlg.cpp @@ -33,7 +33,7 @@ CClientSettingsDlg::CClientSettingsDlg ( CClient* pNCliP, CClientSettings* pNSet setupUi ( this ); #if defined( Q_OS_IOS ) - // IOS needs menu to close + // iOS needs menu to close QMenuBar* pMenu = new QMenuBar ( this ); QAction* action = pMenu->addAction ( tr ( "&Close" ) ); connect ( action, SIGNAL ( triggered() ), this, SLOT ( close() ) ); diff --git a/src/main.cpp b/src/main.cpp index 1205aa44b0..94ee8de2c2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -738,6 +738,7 @@ int main ( int argc, char** argv ) #else # if defined( Q_OS_IOS ) bUseGUI = true; + bIsClient = true; // Client only - TODO: maybe a switch in interface to change to server? // bUseMultithreading = true; QApplication* pApp = new QApplication ( argc, argv ); diff --git a/src/settings.h b/src/settings.h index 1cbc4eec31..ceb46dff29 100644 --- a/src/settings.h +++ b/src/settings.h @@ -120,6 +120,7 @@ class CClientSettings : public CSettings vstrIPAddress ( MAX_NUM_SERVER_ADDR_ITEMS, "" ), iNewClientFaderLevel ( 100 ), iInputBoost ( 1 ), + iBuiltInMicId ( 0 ), iSettingsTab ( SETTING_TAB_AUDIONET ), bConnectDlgShowAllMusicians ( true ), eChannelSortType ( ST_NO_SORT ), @@ -152,6 +153,7 @@ class CClientSettings : public CSettings int iNewClientFaderLevel; int iInputBoost; int iSettingsTab; + int iBuiltInMicId; // 0 for external bool bConnectDlgShowAllMusicians; EChSortType eChannelSortType; int iNumMixerPanelRows; diff --git a/src/socket.cpp b/src/socket.cpp index d7cdb9a699..e4c76cf297 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -40,6 +40,11 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const int UdpSocketAddrLen; uint16_t* UdpPort; + // first store parameters, in case reinit is required (mostly for iOS) + iPortNumber_ = iPortNumber; + iQosNumber_ = iQosNumber; + strServerBindIP_ = strServerBindIP; + #ifdef _WIN32 // for the Windows socket usage we have to start it up first @@ -114,6 +119,12 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const } } +#ifdef Q_OS_IOS + // ignore the broken pipe signal to avoid crash (iOS) + int valueone = 1; + setsockopt ( UdpSocket, SOL_SOCKET, SO_NOSIGPIPE, &valueone, sizeof ( valueone ) ); +#endif + // allocate memory for network receive and send buffer in samples vecbyRecBuf.Init ( MAX_SIZE_BYTES_NETW_BUF ); @@ -170,6 +181,8 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const // Connections ------------------------------------------------------------- // it is important to do the following connections in this class since we // have a thread transition + if ( bIsInitRan ) + return; // we have different connections for client and server if ( bIsClient ) @@ -197,6 +210,8 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); } + + bIsInitRan = true; // QObject::connect once only } void CSocket::Close() @@ -227,6 +242,8 @@ CSocket::~CSocket() void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddress& HostAddr ) { + int status; + uSockAddr UdpSocketAddr; memset ( &UdpSocketAddr, 0, sizeof ( UdpSocketAddr ) ); @@ -241,57 +258,73 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr // char vector in "const char*", for this we first convert the const // uint8_t vector in a read/write uint8_t vector and then do the cast to // const char *) - if ( HostAddr.InetAddr.protocol() == QAbstractSocket::IPv4Protocol ) + + for ( int tries = 0 ; tries < 2 ; tries++ ) // retry loop in case send fails on iOS { - if ( bEnableIPv6 ) + if ( HostAddr.InetAddr.protocol() == QAbstractSocket::IPv4Protocol ) + { + if ( bEnableIPv6 ) + { + // Linux and Mac allow to pass an AF_INET address to a dual-stack socket, + // but Windows does not. So use a V4MAPPED address in an AF_INET6 sockaddr, + // which works on all platforms. + + UdpSocketAddr.sa6.sin6_family = AF_INET6; + UdpSocketAddr.sa6.sin6_port = htons ( HostAddr.iPort ); + + uint32_t* addr = (uint32_t*) &UdpSocketAddr.sa6.sin6_addr; + + addr[0] = 0; + addr[1] = 0; + addr[2] = htonl ( 0xFFFF ); + addr[3] = htonl ( HostAddr.InetAddr.toIPv4Address() ); + + status = sendto ( UdpSocket, + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa6 ) ); + } + else + { + UdpSocketAddr.sa4.sin_family = AF_INET; + UdpSocketAddr.sa4.sin_port = htons ( HostAddr.iPort ); + UdpSocketAddr.sa4.sin_addr.s_addr = htonl ( HostAddr.InetAddr.toIPv4Address() ); + + status = sendto ( UdpSocket, + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa4 ) ); + } + } + else if ( bEnableIPv6 ) { - // Linux and Mac allow to pass an AF_INET address to a dual-stack socket, - // but Windows does not. So use a V4MAPPED address in an AF_INET6 sockaddr, - // which works on all platforms. - UdpSocketAddr.sa6.sin6_family = AF_INET6; UdpSocketAddr.sa6.sin6_port = htons ( HostAddr.iPort ); - - uint32_t* addr = (uint32_t*) &UdpSocketAddr.sa6.sin6_addr; - - addr[0] = 0; - addr[1] = 0; - addr[2] = htonl ( 0xFFFF ); - addr[3] = htonl ( HostAddr.InetAddr.toIPv4Address() ); - - sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa6 ) ); + inet_pton ( AF_INET6, HostAddr.InetAddr.toString().toLocal8Bit().constData(), &UdpSocketAddr.sa6.sin6_addr ); + + status = sendto ( UdpSocket, + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa6 ) ); } - else + + if ( status >= 0 ) { - UdpSocketAddr.sa4.sin_family = AF_INET; - UdpSocketAddr.sa4.sin_port = htons ( HostAddr.iPort ); - UdpSocketAddr.sa4.sin_addr.s_addr = htonl ( HostAddr.InetAddr.toIPv4Address() ); - - sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa4 ) ); + break; // do not retry if success } - } - else if ( bEnableIPv6 ) - { - UdpSocketAddr.sa6.sin6_family = AF_INET6; - UdpSocketAddr.sa6.sin6_port = htons ( HostAddr.iPort ); - inet_pton ( AF_INET6, HostAddr.InetAddr.toString().toLocal8Bit().constData(), &UdpSocketAddr.sa6.sin6_addr ); - - sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa6 ) ); + +#ifdef Q_OS_IOS + // qDebug("Socket send exception - mostly happens in iOS when returning from idle"); + Init ( iPortNumber_, iQosNumber_, strServerBindIP_ ); // reinit + + // loop back to retry +#endif } } } diff --git a/src/socket.h b/src/socket.h index d41a343881..fb412cfa3c 100644 --- a/src/socket.h +++ b/src/socket.h @@ -79,7 +79,11 @@ class CSocket : public QObject void Close(); protected: - void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); + void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); + quint16 iPortNumber_; + quint16 iQosNumber_; + QString strServerBindIP_; + bool bIsInitRan; #ifdef _WIN32 SOCKET UdpSocket; From 3e8ddc431f8b7a343344baa43e02f147fd3384c9 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 14:49:00 +0200 Subject: [PATCH 08/14] iOS code review modified: ios/sound.h modified: src/socket.cpp --- ios/sound.h | 11 +++-- ios/sound.mm | 98 ++++++++++++++++++---------------------- src/socket.cpp | 118 +++++++++++++++++++++++++------------------------ src/socket.h | 7 ++- 4 files changed, 113 insertions(+), 121 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index aeeec03f79..b886d1c797 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -23,7 +23,6 @@ \******************************************************************************/ #pragma once -#include #include "soundbase.h" #include "global.h" @@ -56,7 +55,11 @@ class CSound : public CSoundBase bool isInitialized; protected: - MIDIPortRef midiInPortRef; - - QMutex Mutex; + void checkStatus ( int status ); + static OSStatus recordingCallback ( void* inRefCon, + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ); }; diff --git a/ios/sound.mm b/ios/sound.mm index e827f29256..9d315ef376 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -27,28 +27,39 @@ #define kOutputBus 0 #define kInputBus 1 -void checkStatus ( int status ) +/* Implementation *************************************************************/ +CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), + void* arg, + const QString& strMIDISetup, + const bool, + const QString& ) : + CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), + isInitialized ( false ) { - if ( status ) + try { - printf ( "Status not 0! %d\n", status ); - } -} + NSError* audioSessionError = nil; -void checkStatus ( int status, char* s ) -{ - if ( status ) + [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; + [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { + if ( granted ) + { + // ok + } + }]; + [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; + } + catch ( const CGenErr& generr ) { - printf ( "Status not 0! %d - %s \n", status, s ); + qDebug ( "Sound exception ...." ); } } /** - This callback is called when sound card needs output data to play. And because Jamulus use the same buffer to store input and output data (input is - sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the - output fetched from server to ioData, which will then be played. + This callback is called when sound card needs output data to play. + And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. */ -static OSStatus recordingCallback ( void* inRefCon, +OSStatus CSound::recordingCallback ( void* inRefCon, AudioUnitRenderActionFlags* ioActionFlags, const AudioTimeStamp* inTimeStamp, UInt32 inBusNumber, @@ -69,14 +80,10 @@ static OSStatus recordingCallback ( void* inRefCon, bufferList.mNumberBuffers = 1; bufferList.mBuffers[0] = buffer; - // Then: // Obtain recorded samples - OSStatus status; - // Calling Unit Render to store input data to bufferList - status = AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); - // checkStatus(status, (char *)" Just called AudioUnitRender "); + AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); // Now, we have the samples we just read sitting in buffers in bufferList // Process the new data @@ -112,42 +119,12 @@ static OSStatus recordingCallback ( void* inRefCon, pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); } -/* Implementation *************************************************************/ -CSound::CSound ( void ( *fpNewProcessCallback ) ( CVector& psData, void* arg ), - void* arg, - const QString& strMIDISetup, - const bool, - const QString& ) : - CSoundBase ( "CoreAudio iOS", fpNewProcessCallback, arg, strMIDISetup ), - midiInPortRef ( static_cast ( NULL ) ) -{ - try - { - NSError* audioSessionError = nil; - - [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; - [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { - if ( granted ) - { - // ok - } - }]; - [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; - } - catch ( const CGenErr& generr ) - { - qDebug ( "Sound exception ...." ); // This try-catch seems to fix Connect button crash - } - isInitialized = false; -} - +// TODO - CSound::Init is called multiple times at launch to verify device capabilities. +// For iOS though, Init takes long, so should better reduce those inits for iOS (Now: 9 times/launch). int CSound::Init ( const int iCoreAudioBufferSizeMono ) { try { - // printf("Init sound ..."); <= to check the number of Sound inits at launch - // init base class - // CSoundBase::Init ( iCoreAudioBufferSizeMono ); this does nothing this->iCoreAudioBufferSizeMono = iCoreAudioBufferSizeMono; // set internal buffer size value and calculate stereo buffer size @@ -166,11 +143,11 @@ static OSStatus recordingCallback ( void* inRefCon, AVAudioSessionCategoryOptionAllowAirPlay error:&error]; - // NGOCDH - using values from jamulus settings 64 = 2.67ms/2 + // using values from jamulus settings 64 = 2.67ms/2 NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; - // set the session's sample rate 48000 - the only supported by Jamulus ? + // set the session's sample rate 48000 - the only supported by Jamulus [sessionInstance setPreferredSampleRate:48000 error:&error]; [[AVAudioSession sharedInstance] setActive:YES error:&error]; @@ -247,7 +224,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Init exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Init exception ...." ); } return iCoreAudioBufferSizeMono; @@ -264,7 +241,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Start exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Start exception ...." ); } } @@ -277,7 +254,7 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound Stop exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound Stop exception ...." ); } // call base class CSoundBase::Stop(); @@ -294,6 +271,7 @@ static OSStatus recordingCallback ( void* inRefCon, builtinmic = false; // try external device AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device if ( builtinmic ) { @@ -307,6 +285,14 @@ static OSStatus recordingCallback ( void* inRefCon, } catch ( const CGenErr& generr ) { - qDebug ( "Sound dev change exception ...." ); // This try-catch seems to fix Connect button crash + qDebug ( "Sound dev change exception ...." ); + } +} + +void CSound::checkStatus ( int status ) +{ + if ( status ) + { + printf ( "Status not 0! %d\n", status ); } } diff --git a/src/socket.cpp b/src/socket.cpp index e4c76cf297..4dac0316f3 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -33,7 +33,46 @@ #endif /* Implementation *************************************************************/ -void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) + +// Connections ------------------------------------------------------------- +// it is important to do the following connections in this class since we +// have a thread transition + +// we have different connections for client and server, created after Init in corresponding constructor + +CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pChannel ( pNewChannel ), + bIsClient ( true ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // client connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); + + QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); +} + +CSocket::CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : + pServer ( pNServP ), + bIsClient ( false ), + bJitterBufferOK ( true ) +{ + Init ( iPortNumber, iQosNumber, strServerBindIP ); + + // server connections: + QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); + + QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); + + QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pServer, &CServer::OnNewConnection ); + + QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); +} + +void CSocket::Init ( const quint16 iNewPortNumber, const quint16 iNewQosNumber, const QString& strNewServerBindIP ) { uSockAddr UdpSocketAddr; @@ -41,9 +80,9 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const uint16_t* UdpPort; // first store parameters, in case reinit is required (mostly for iOS) - iPortNumber_ = iPortNumber; - iQosNumber_ = iQosNumber; - strServerBindIP_ = strServerBindIP; + iPortNumber = iNewPortNumber; + iQosNumber = iNewQosNumber; + strServerBindIP = strNewServerBindIP; #ifdef _WIN32 // for the Windows socket usage we have to start it up first @@ -177,41 +216,6 @@ void CSocket::Init ( const quint16 iPortNumber, const quint16 iQosNumber, const // we cannot bind socket, throw error throw CGenErr ( "Cannot bind the socket (maybe the software is already running).", "Network Error" ); } - - // Connections ------------------------------------------------------------- - // it is important to do the following connections in this class since we - // have a thread transition - if ( bIsInitRan ) - return; - - // we have different connections for client and server - if ( bIsClient ) - { - // client connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pChannel, &CChannel::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pChannel, &CChannel::OnProtcolCLMessageReceived ); - - QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); - } - else - { - // server connections: - - QObject::connect ( this, &CSocket::ProtcolMessageReceived, pServer, &CServer::OnProtcolMessageReceived ); - - QObject::connect ( this, &CSocket::ProtcolCLMessageReceived, pServer, &CServer::OnProtcolCLMessageReceived ); - - QObject::connect ( this, - static_cast ( &CSocket::NewConnection ), - pServer, - &CServer::OnNewConnection ); - - QObject::connect ( this, &CSocket::ServerFull, pServer, &CServer::OnServerFull ); - } - - bIsInitRan = true; // QObject::connect once only } void CSocket::Close() @@ -259,7 +263,7 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr // uint8_t vector in a read/write uint8_t vector and then do the cast to // const char *) - for ( int tries = 0 ; tries < 2 ; tries++ ) // retry loop in case send fails on iOS + for ( int tries = 0; tries < 2; tries++ ) // retry loop in case send fails on iOS { if ( HostAddr.InetAddr.protocol() == QAbstractSocket::IPv4Protocol ) { @@ -280,11 +284,11 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr addr[3] = htonl ( HostAddr.InetAddr.toIPv4Address() ); status = sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa6 ) ); + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa6 ) ); } else { @@ -293,11 +297,11 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr UdpSocketAddr.sa4.sin_addr.s_addr = htonl ( HostAddr.InetAddr.toIPv4Address() ); status = sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa4 ) ); + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa4 ) ); } } else if ( bEnableIPv6 ) @@ -307,21 +311,21 @@ void CSocket::SendPacket ( const CVector& vecbySendBuf, const CHostAddr inet_pton ( AF_INET6, HostAddr.InetAddr.toString().toLocal8Bit().constData(), &UdpSocketAddr.sa6.sin6_addr ); status = sendto ( UdpSocket, - (const char*) &( (CVector) vecbySendBuf )[0], - iVecSizeOut, - 0, - &UdpSocketAddr.sa, - sizeof ( UdpSocketAddr.sa6 ) ); + (const char*) &( (CVector) vecbySendBuf )[0], + iVecSizeOut, + 0, + &UdpSocketAddr.sa, + sizeof ( UdpSocketAddr.sa6 ) ); } if ( status >= 0 ) { - break; // do not retry if success + break; // do not retry if success } #ifdef Q_OS_IOS // qDebug("Socket send exception - mostly happens in iOS when returning from idle"); - Init ( iPortNumber_, iQosNumber_, strServerBindIP_ ); // reinit + Init ( iPortNumber, iQosNumber, strServerBindIP ); // reinit // loop back to retry #endif diff --git a/src/socket.h b/src/socket.h index fb412cfa3c..ba75293dfd 100644 --- a/src/socket.h +++ b/src/socket.h @@ -80,10 +80,9 @@ class CSocket : public QObject protected: void Init ( const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ); - quint16 iPortNumber_; - quint16 iQosNumber_; - QString strServerBindIP_; - bool bIsInitRan; + quint16 iPortNumber; + quint16 iQosNumber; + QString strServerBindIP; #ifdef _WIN32 SOCKET UdpSocket; From 194983322516b7f793856c52d0ffac9c0cd3bde7 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Mon, 14 Jun 2021 14:49:00 +0200 Subject: [PATCH 09/14] iOS code review modified: ios/sound.h modified: ios/sound.mm modified: src/socket.cpp modified: src/socket.h --- src/socket.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/socket.cpp b/src/socket.cpp index 4dac0316f3..a8d32042a0 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -214,7 +214,9 @@ void CSocket::Init ( const quint16 iNewPortNumber, const quint16 iNewQosNumber, if ( !bSuccess ) { // we cannot bind socket, throw error - throw CGenErr ( "Cannot bind the socket (maybe the software is already running).", "Network Error" ); + throw CGenErr ( "Cannot bind the socket (maybe " + "the software is already running).", + "Network Error" ); } } From 321cae04b59b50dbd8f665b55a9429a503de9d21 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Wed, 16 Jun 2021 09:15:32 +0200 Subject: [PATCH 10/14] iOS sound - no more malloc during sound processing modified: ios/sound.mm --- ios/sound.h | 3 +++ ios/sound.mm | 50 +++++++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index b886d1c797..15c15e3b36 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -38,6 +38,7 @@ class CSound : public CSoundBase const QString& strMIDISetup, const bool, const QString& ); + ~CSound(); virtual int Init ( const int iNewPrefMonoBufferSize ); virtual void Start(); @@ -55,6 +56,8 @@ class CSound : public CSoundBase bool isInitialized; protected: + AudioBuffer buffer; + AudioBufferList bufferList; void checkStatus ( int status ); static OSStatus recordingCallback ( void* inRefCon, AudioUnitRenderActionFlags* ioActionFlags, diff --git a/ios/sound.mm b/ios/sound.mm index 9d315ef376..49ffbf7087 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -46,13 +46,27 @@ { // ok } + else + { + // TODO - alert user + } }]; [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } catch ( const CGenErr& generr ) { - qDebug ( "Sound exception ...." ); + QMessageBox::warning( nullptr, "Sound exception", generr.GetErrorText() ); } + + buffer.mNumberChannels = 2; + buffer.mData = malloc ( 256 * sizeof ( Float32 ) * 2 ); // max size + bufferList.mNumberBuffers = 1; + bufferList.mBuffers[0] = buffer; +} + +CSound::~CSound () +{ + free ( buffer.mData ); } /** @@ -69,28 +83,18 @@ And because Jamulus uses the same buffer to store input and output data (input i CSound* pSound = static_cast ( inRefCon ); - AudioBuffer buffer; - - buffer.mNumberChannels = 2; - buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * buffer.mNumberChannels; - buffer.mData = malloc ( buffer.mDataByteSize ); - - // Put buffer in a AudioBufferList - AudioBufferList bufferList; - bufferList.mNumberBuffers = 1; - bufferList.mBuffers[0] = buffer; + // setting up temp buffer + pSound->buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * pSound->buffer.mNumberChannels; + pSound->bufferList.mBuffers[0] = pSound->buffer; // Obtain recorded samples // Calling Unit Render to store input data to bufferList - AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &bufferList ); + AudioUnitRender ( pSound->audioUnit, ioActionFlags, inTimeStamp, 1, inNumberFrames, &pSound->bufferList ); // Now, we have the samples we just read sitting in buffers in bufferList // Process the new data - pSound->processBufferList ( &bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList - - // release the malloc'ed data in the buffer we created earlier - free ( bufferList.mBuffers[0].mData ); + pSound->processBufferList ( &pSound->bufferList, pSound ); // THIS IS WHERE vecsStereo is filled with data from bufferList Float32* pData = (Float32*) ( ioData->mBuffers[0].mData ); @@ -144,11 +148,11 @@ And because Jamulus uses the same buffer to store input and output data (input i error:&error]; // using values from jamulus settings 64 = 2.67ms/2 - NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / 48000.0; // yeah it's math + NSTimeInterval bufferDuration = iCoreAudioBufferSizeMono / Float32 ( SYSTEM_SAMPLE_RATE_HZ ); // yeah it's math [sessionInstance setPreferredIOBufferDuration:bufferDuration error:&error]; // set the session's sample rate 48000 - the only supported by Jamulus - [sessionInstance setPreferredSampleRate:48000 error:&error]; + [sessionInstance setPreferredSampleRate:SYSTEM_SAMPLE_RATE_HZ error:&error]; [[AVAudioSession sharedInstance] setActive:YES error:&error]; OSStatus status; @@ -179,7 +183,7 @@ And because Jamulus uses the same buffer to store input and output data (input i // Describe format AudioStreamBasicDescription audioFormat; - audioFormat.mSampleRate = 48000.00; + audioFormat.mSampleRate = SYSTEM_SAMPLE_RATE_HZ; audioFormat.mFormatID = kAudioFormatLinearPCM; audioFormat.mFormatFlags = kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; audioFormat.mFramesPerPacket = 1; @@ -224,7 +228,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Init exception ...." ); + QMessageBox::warning( nullptr, "Sound init exception", generr.GetErrorText() ); } return iCoreAudioBufferSizeMono; @@ -241,7 +245,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Start exception ...." ); + QMessageBox::warning( nullptr, "Sound start exception", generr.GetErrorText() ); } } @@ -254,7 +258,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound Stop exception ...." ); + QMessageBox::warning( nullptr, "Sound stop exception", generr.GetErrorText() ); } // call base class CSoundBase::Stop(); @@ -285,7 +289,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - qDebug ( "Sound dev change exception ...." ); + QMessageBox::warning( nullptr, "Sound device change exception", generr.GetErrorText() ); } } From 6848631f5dc3ceeb1660367b58e57c1142efdcc8 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Sat, 19 Jun 2021 09:31:20 +0200 Subject: [PATCH 11/14] Remove iOS device switch feature to keep code consistent --- ios/sound.h | 1 - ios/sound.mm | 29 ----------------------------- src/client.cpp | 9 --------- src/client.h | 6 ------ src/clientdlg.cpp | 11 ----------- src/clientdlg.h | 14 -------------- src/settings.h | 2 -- 7 files changed, 72 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index 15c15e3b36..904b6a2ad2 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -44,7 +44,6 @@ class CSound : public CSoundBase virtual void Start(); virtual void Stop(); virtual void processBufferList ( AudioBufferList*, CSound* ); - virtual void SetInputDeviceId ( int deviceid ); AudioUnit audioUnit; diff --git a/ios/sound.mm b/ios/sound.mm index 49ffbf7087..a04e57a145 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -264,35 +264,6 @@ And because Jamulus uses the same buffer to store input and output data (input i CSoundBase::Stop(); } -void CSound::SetInputDeviceId ( int deviceid ) -{ - try - { - NSError* error = nil; - bool builtinmic = true; - - if ( deviceid == 0 ) - builtinmic = false; // try external device - - AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; - - // assumming iOS only has max 2 inputs: 0 for builtin mic and 1 for external device - if ( builtinmic ) - { - [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; - } - else - { - unsigned long lastInput = sessionInstance.availableInputs.count - 1; - [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; - } - } - catch ( const CGenErr& generr ) - { - QMessageBox::warning( nullptr, "Sound device change exception", generr.GetErrorText() ); - } -} - void CSound::checkStatus ( int status ) { if ( status ) diff --git a/src/client.cpp b/src/client.cpp index 20237b66fe..decc43a768 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -53,7 +53,6 @@ CClient::CClient ( const quint16 iPortNumber, bReverbOnLeftChan ( false ), iReverbLevel ( 0 ), iInputBoost ( 1 ), - iBuiltInMicId ( 0 ), iSndCrdPrefFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), iSndCrdFrameSizeFactor ( FRAME_SIZE_FACTOR_DEFAULT ), bSndCrdConversionBufferRequired ( false ), @@ -1253,11 +1252,3 @@ int CClient::EstimatedOverallDelay ( const int iPingTimeMs ) return MathUtils::round ( fTotalBufferDelayMs + iPingTimeMs ); } - -void CClient::SetInputDeviceId ( const int deviceid ) -{ -#if defined( Q_OS_IOS ) - // iOS only - Sound.SetInputDeviceId ( deviceid ); -#endif -} diff --git a/src/client.h b/src/client.h index a7b731d45a..ca82328080 100644 --- a/src/client.h +++ b/src/client.h @@ -241,9 +241,6 @@ class CClient : public QObject void SetInputBoost ( const int iNewBoost ) { iInputBoost = iNewBoost; } - void SetBuiltInMicId ( const int iNewMicId ) { iBuiltInMicId = iNewMicId; } - int GetBuiltInMicId() { return iBuiltInMicId; } - void SetRemoteInfo() { Channel.SetRemoteInfo ( ChannelInfo ); } void CreateChatTextMes ( const QString& strChatText ) { Channel.CreateChatTextMes ( strChatText ); } @@ -268,8 +265,6 @@ class CClient : public QObject Channel.GetBufErrorRates ( vecErrRates, dLimit, dMaxUpLimit ); } - void SetInputDeviceId ( const int deviceid ); // for mobile devices - 0 for external devices - // settings CChannelCoreInfo ChannelInfo; QString strClientName; @@ -330,7 +325,6 @@ class CClient : public QObject int iReverbLevel; CAudioReverb AudioReverb; int iInputBoost; - int iBuiltInMicId; int iSndCrdPrefFrameSizeFactor; int iSndCrdFrameSizeFactor; diff --git a/src/clientdlg.cpp b/src/clientdlg.cpp index 14034bc2cc..9de0b354f0 100644 --- a/src/clientdlg.cpp +++ b/src/clientdlg.cpp @@ -386,17 +386,6 @@ CClientDlg::CClientDlg ( CClient* pNCliP, pMenu->addMenu ( pSettingsMenu ); pMenu->addMenu ( new CHelpMenu ( true, this ) ); -#if defined( Q_OS_IOS ) - // Mobile input -------------------------------------------------------------- - QMenu* pMobileMenu = new QMenu ( tr ( "&Mobile input" ), this ); - - pMobileMenu->addAction ( tr ( "Builtin Mic" ), this, SLOT ( setBuiltinMic() ) ); - - pMobileMenu->addAction ( tr ( "Auto" ), this, SLOT ( unsetBuiltinMic() ) ); - - pMenu->addMenu ( pMobileMenu ); -#endif - // Now tell the layout about the menu layout()->setMenuBar ( pMenu ); diff --git a/src/clientdlg.h b/src/clientdlg.h index 6a3cef6e46..5437fb8b2a 100644 --- a/src/clientdlg.h +++ b/src/clientdlg.h @@ -233,20 +233,6 @@ public slots: void accept() { close(); } // introduced by pljones - void setBuiltinMic() - { -#if defined( Q_OS_IOS ) - pClient->SetInputDeviceId ( 1 ); -#endif - } - - void unsetBuiltinMic() - { -#if defined( Q_OS_IOS ) - pClient->SetInputDeviceId ( 0 ); -#endif - } - signals: void SendTabChange ( int iTabIdx ); }; diff --git a/src/settings.h b/src/settings.h index ceb46dff29..1cbc4eec31 100644 --- a/src/settings.h +++ b/src/settings.h @@ -120,7 +120,6 @@ class CClientSettings : public CSettings vstrIPAddress ( MAX_NUM_SERVER_ADDR_ITEMS, "" ), iNewClientFaderLevel ( 100 ), iInputBoost ( 1 ), - iBuiltInMicId ( 0 ), iSettingsTab ( SETTING_TAB_AUDIONET ), bConnectDlgShowAllMusicians ( true ), eChannelSortType ( ST_NO_SORT ), @@ -153,7 +152,6 @@ class CClientSettings : public CSettings int iNewClientFaderLevel; int iInputBoost; int iSettingsTab; - int iBuiltInMicId; // 0 for external bool bConnectDlgShowAllMusicians; EChSortType eChannelSortType; int iNumMixerPanelRows; From 85668ea5bb28c4538c7b7731bc023fcbd01978d3 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Tue, 29 Jun 2021 10:52:04 +0200 Subject: [PATCH 12/14] iOS switch device feature modified: ios/sound.mm --- ios/sound.h | 6 +++ ios/sound.mm | 121 +++++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 103 insertions(+), 24 deletions(-) diff --git a/ios/sound.h b/ios/sound.h index 904b6a2ad2..f6bdee07f3 100644 --- a/ios/sound.h +++ b/ios/sound.h @@ -55,6 +55,10 @@ class CSound : public CSoundBase bool isInitialized; protected: + virtual QString LoadAndInitializeDriver ( QString strDriverName, bool ); + void GetAvailableInOutDevices(); + void SwitchDevice ( QString strDriverName ); + AudioBuffer buffer; AudioBufferList bufferList; void checkStatus ( int status ); @@ -64,4 +68,6 @@ class CSound : public CSoundBase UInt32 inBusNumber, UInt32 inNumberFrames, AudioBufferList* ioData ); + + QMutex Mutex; }; diff --git a/ios/sound.mm b/ios/sound.mm index a04e57a145..eba38b8a11 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -42,50 +42,48 @@ [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayAndRecord error:&audioSessionError]; [[AVAudioSession sharedInstance] requestRecordPermission:^( BOOL granted ) { - if ( granted ) - { - // ok - } - else - { - // TODO - alert user - } + if ( granted ) + { + // ok + } + else + { + // TODO - alert user + } }]; [[AVAudioSession sharedInstance] setMode:AVAudioSessionModeMeasurement error:&audioSessionError]; } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound exception", generr.GetErrorText() ); } buffer.mNumberChannels = 2; - buffer.mData = malloc ( 256 * sizeof ( Float32 ) * 2 ); // max size + buffer.mData = malloc ( 256 * sizeof ( Float32 ) * buffer.mNumberChannels ); // max size bufferList.mNumberBuffers = 1; bufferList.mBuffers[0] = buffer; } -CSound::~CSound () -{ - free ( buffer.mData ); -} +CSound::~CSound() { free ( buffer.mData ); } /** This callback is called when sound card needs output data to play. - And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. + And because Jamulus uses the same buffer to store input and output data (input is sent to server, then output is fetched from server), we actually + use the output callback to read inputdata first, process it, and then copy the output fetched from server to ioData, which will then be played. */ OSStatus CSound::recordingCallback ( void* inRefCon, - AudioUnitRenderActionFlags* ioActionFlags, - const AudioTimeStamp* inTimeStamp, - UInt32 inBusNumber, - UInt32 inNumberFrames, - AudioBufferList* ioData ) + AudioUnitRenderActionFlags* ioActionFlags, + const AudioTimeStamp* inTimeStamp, + UInt32 inBusNumber, + UInt32 inNumberFrames, + AudioBufferList* ioData ) { CSound* pSound = static_cast ( inRefCon ); // setting up temp buffer pSound->buffer.mDataByteSize = pSound->iCoreAudioBufferSizeMono * sizeof ( Float32 ) * pSound->buffer.mNumberChannels; - pSound->bufferList.mBuffers[0] = pSound->buffer; + pSound->bufferList.mBuffers[0] = pSound->buffer; // Obtain recorded samples @@ -224,11 +222,29 @@ And because Jamulus uses the same buffer to store input and output data (input i status = AudioUnitInitialize ( audioUnit ); checkStatus ( status ); + SwitchDevice ( strCurDevName ); + + if ( !isInitialized ) + { + [[NSNotificationCenter defaultCenter] addObserverForName:AVAudioSessionRouteChangeNotification + object:nil + queue:nil + usingBlock:^( NSNotification* notification ) { + UInt8 reason = + [[notification.userInfo valueForKey:AVAudioSessionRouteChangeReasonKey] intValue]; + if ( reason == AVAudioSessionRouteChangeReasonNewDeviceAvailable or + reason == AVAudioSessionRouteChangeReasonOldDeviceUnavailable ) + { + emit ReinitRequest ( RS_RELOAD_RESTART_AND_INIT ); // reload the available devices frame + } + }]; + } + isInitialized = true; } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound init exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound init exception", generr.GetErrorText() ); } return iCoreAudioBufferSizeMono; @@ -245,7 +261,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound start exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound start exception", generr.GetErrorText() ); } } @@ -258,7 +274,7 @@ And because Jamulus uses the same buffer to store input and output data (input i } catch ( const CGenErr& generr ) { - QMessageBox::warning( nullptr, "Sound stop exception", generr.GetErrorText() ); + QMessageBox::warning ( nullptr, "Sound stop exception", generr.GetErrorText() ); } // call base class CSoundBase::Stop(); @@ -271,3 +287,60 @@ And because Jamulus uses the same buffer to store input and output data (input i printf ( "Status not 0! %d\n", status ); } } + +QString CSound::LoadAndInitializeDriver ( QString strDriverName, bool ) +{ + // secure lNumDevs/strDriverNames access + QMutexLocker locker ( &Mutex ); + + // reload the driver list of available sound devices + GetAvailableInOutDevices(); + + // store the current name of the driver + strCurDevName = strDriverName; + + return ""; +} + +void CSound::GetAvailableInOutDevices() +{ + // always add system default devices for input and output as first entry + lNumDevs = 1; + strDriverNames[0] = "System Default In/Out Devices"; + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + if ( sessionInstance.availableInputs.count > 1 ) + { + lNumDevs = 2; + strDriverNames[1] = "in: Built-in Mic/out: System Default"; + } +} + +void CSound::SwitchDevice ( QString strDriverName ) +{ + // find driver index from given driver name + int iDriverIdx = INVALID_INDEX; // initialize with an invalid index + + for ( int i = 0; i < MAX_NUMBER_SOUND_CARDS; i++ ) + { + if ( strDriverName.compare ( strDriverNames[i] ) == 0 ) + { + iDriverIdx = i; + } + } + + NSError* error = nil; + + AVAudioSession* sessionInstance = [AVAudioSession sharedInstance]; + + if ( iDriverIdx == 0 ) // system default device + { + unsigned long lastInput = sessionInstance.availableInputs.count - 1; + [sessionInstance setPreferredInput:sessionInstance.availableInputs[lastInput] error:&error]; + } + else // built-in mic + { + [sessionInstance setPreferredInput:sessionInstance.availableInputs[0] error:&error]; + } +} From f91f139f087c59b4dd83a01928f632c4ccbe853d Mon Sep 17 00:00:00 2001 From: Tony Mountifield Date: Mon, 6 Sep 2021 18:00:57 +0100 Subject: [PATCH 13/14] Fixup rebase mistakes --- src/socket.cpp | 10 ++++++---- src/socket.h | 19 ++----------------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/socket.cpp b/src/socket.cpp index a8d32042a0..3a6622f91a 100644 --- a/src/socket.cpp +++ b/src/socket.cpp @@ -40,10 +40,11 @@ // we have different connections for client and server, created after Init in corresponding constructor -CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : +CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ) : pChannel ( pNewChannel ), bIsClient ( true ), - bJitterBufferOK ( true ) + bJitterBufferOK ( true ), + bEnableIPv6 ( bEnableIPv6 ) { Init ( iPortNumber, iQosNumber, strServerBindIP ); @@ -55,10 +56,11 @@ CSocket::CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint QObject::connect ( this, static_cast ( &CSocket::NewConnection ), pChannel, &CChannel::OnNewConnection ); } -CSocket::CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP ) : +CSocket::CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ) : pServer ( pNServP ), bIsClient ( false ), - bJitterBufferOK ( true ) + bJitterBufferOK ( true ), + bEnableIPv6 ( bEnableIPv6 ) { Init ( iPortNumber, iQosNumber, strServerBindIP ); diff --git a/src/socket.h b/src/socket.h index ba75293dfd..c29df14d9d 100644 --- a/src/socket.h +++ b/src/socket.h @@ -53,23 +53,8 @@ class CSocket : public QObject Q_OBJECT public: - CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ) : - pChannel ( pNewChannel ), - bIsClient ( true ), - bJitterBufferOK ( true ), - bEnableIPv6 ( bEnableIPv6 ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } - - CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ) : - pServer ( pNServP ), - bIsClient ( false ), - bJitterBufferOK ( true ), - bEnableIPv6 ( bEnableIPv6 ) - { - Init ( iPortNumber, iQosNumber, strServerBindIP ); - } + CSocket ( CChannel* pNewChannel, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ); + CSocket ( CServer* pNServP, const quint16 iPortNumber, const quint16 iQosNumber, const QString& strServerBindIP, bool bEnableIPv6 ); virtual ~CSocket(); From f75fd8efe9ce48ee9d731e7d80fdb8c42a2daa21 Mon Sep 17 00:00:00 2001 From: ngocdh Date: Tue, 7 Sep 2021 09:36:23 +0200 Subject: [PATCH 14/14] iOS sound - Typo fix and add break in loop --- ios/sound.mm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ios/sound.mm b/ios/sound.mm index eba38b8a11..b2366b51f6 100644 --- a/ios/sound.mm +++ b/ios/sound.mm @@ -185,7 +185,7 @@ And because Jamulus uses the same buffer to store input and output data (input i audioFormat.mFormatID = kAudioFormatLinearPCM; audioFormat.mFormatFlags = kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; audioFormat.mFramesPerPacket = 1; - audioFormat.mChannelsPerFrame = 2; // steoreo, so 2 interleaved channels + audioFormat.mChannelsPerFrame = 2; // stereo, so 2 interleaved channels audioFormat.mBitsPerChannel = 32; // sizeof float32 audioFormat.mBytesPerPacket = 8; // (sizeof float32) * 2 channels audioFormat.mBytesPerFrame = 8; //(sizeof float32) * 2 channels @@ -327,6 +327,7 @@ And because Jamulus uses the same buffer to store input and output data (input i if ( strDriverName.compare ( strDriverNames[i] ) == 0 ) { iDriverIdx = i; + break; } }