From 69192ce5131fea6e67118a3b8d4b6b970635d4b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=8B=E4=BB=95=E9=92=8A?= Date: Tue, 22 Jun 2021 00:40:05 +0800 Subject: [PATCH 1/4] add audio support --- CaptureEncoder/Audio.cs | 309 +++++++++++++++++++++++++++ CaptureEncoder/CaptureEncoder.csproj | 3 +- CaptureEncoder/Encoder.cs | 50 ++++- SimpleRecorder/SimpleRecorder.csproj | 2 +- 4 files changed, 352 insertions(+), 12 deletions(-) create mode 100644 CaptureEncoder/Audio.cs diff --git a/CaptureEncoder/Audio.cs b/CaptureEncoder/Audio.cs new file mode 100644 index 0000000..ba3ce10 --- /dev/null +++ b/CaptureEncoder/Audio.cs @@ -0,0 +1,309 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Windows.Media.Audio; +using Windows.Media.Core; +using Windows.Media.MediaProperties; +using Windows.Media.Streaming.Adaptive; +using Windows.Storage; +using Windows.Storage.Pickers; + +namespace CaptureEncoder +{ + public sealed class MyAudioGraphPlayer + { + + public AudioGraph audioGraph; + public MyAudioGraphPlayer() + { + + + + } + + + + + + /// + /// 初始化音频图 + /// + /// + public async Task InitAudioGraph() + { + AudioGraphSettings settings = new AudioGraphSettings(Windows.Media.Render.AudioRenderCategory.Media); + CreateAudioGraphResult result = await AudioGraph.CreateAsync(settings); + if (result.Status != AudioGraphCreationStatus.Success) + { + Debug.WriteLine("AudioGraph creation error: " + result.Status.ToString()); + } + audioGraph = result.Graph; + audioGraph.UnrecoverableErrorOccurred += async (sender, args) => + { + if (sender == audioGraph && args.Error != AudioGraphUnrecoverableError.None) + { + Debug.WriteLine("The audio graph encountered and unrecoverable error."); + audioGraph.Stop(); + audioGraph.Dispose(); + await InitAudioGraph(); + } + }; + } + + + + + + #region 收起 + + AudioDeviceInputNode deviceInputNode; + + private async Task CreateDeviceInputNode() + { + // Create a device output node + CreateAudioDeviceInputNodeResult result = await audioGraph.CreateDeviceInputNodeAsync(Windows.Media.Capture.MediaCategory.Media); + + if (result.Status != AudioDeviceNodeCreationStatus.Success) + { + // Cannot create device output node + //ShowErrorMessage(result.Status.ToString()); + return; + } + + deviceInputNode = result.DeviceInputNode; + } + + + + #endregion + + + + + public AudioDeviceOutputNode deviceOutputNode; + public async Task CreateDeviceOutputNode() + { + // Create a device output node + CreateAudioDeviceOutputNodeResult result = await audioGraph.CreateDeviceOutputNodeAsync(); + + if (result.Status != AudioDeviceNodeCreationStatus.Success) + { + // Cannot create device output node + //ShowErrorMessage(result.Status.ToString()); + return null; + } + + return result.DeviceOutputNode; + } + + + private async Task CreateFileInputNode() + { + if (audioGraph == null) + return null; + + FileOpenPicker filePicker = new FileOpenPicker(); + filePicker.SuggestedStartLocation = PickerLocationId.MusicLibrary; + filePicker.FileTypeFilter.Add(".mp3"); + filePicker.FileTypeFilter.Add(".wav"); + filePicker.FileTypeFilter.Add(".wma"); + filePicker.FileTypeFilter.Add(".m4a"); + filePicker.ViewMode = PickerViewMode.Thumbnail; + StorageFile file = await filePicker.PickSingleFileAsync(); + + // File can be null if cancel is hit in the file picker + if (file == null) + { + return null; + } + CreateAudioFileInputNodeResult result = await audioGraph.CreateFileInputNodeAsync(file); + + if (result.Status != AudioFileNodeCreationStatus.Success) + { + //ShowErrorMessage(result.Status.ToString()); + } + + return result.FileInputNode; + } + + + + + + + public MediaSourceAudioInputNode mediaSourceInputNode; + + public async Task CreateMediaSourceInputNode(System.Uri contentUri) + { + if (audioGraph == null) + return; + + var adaptiveMediaSourceResult = await AdaptiveMediaSource.CreateFromUriAsync(contentUri); + if (adaptiveMediaSourceResult.Status != AdaptiveMediaSourceCreationStatus.Success) + { + Debug.WriteLine("Failed to create AdaptiveMediaSource"); + + + + return; + } + + var mediaSource = MediaSource.CreateFromAdaptiveMediaSource(adaptiveMediaSourceResult.MediaSource); + CreateMediaSourceAudioInputNodeResult mediaSourceAudioInputNodeResult = + await audioGraph.CreateMediaSourceAudioInputNodeAsync(mediaSource); + + if (mediaSourceAudioInputNodeResult.Status != MediaSourceAudioInputNodeCreationStatus.Success) + { + switch (mediaSourceAudioInputNodeResult.Status) + { + case MediaSourceAudioInputNodeCreationStatus.FormatNotSupported: + Debug.WriteLine("The MediaSource uses an unsupported format"); + break; + case MediaSourceAudioInputNodeCreationStatus.NetworkError: + Debug.WriteLine("The MediaSource requires a network connection and a network-related error occurred"); + break; + case MediaSourceAudioInputNodeCreationStatus.UnknownFailure: + default: + Debug.WriteLine("An unknown error occurred while opening the MediaSource"); + break; + } + return; + } + + mediaSourceInputNode = mediaSourceAudioInputNodeResult.Node; + + + mediaSourceInputNode.MediaSourceCompleted += (s, e) => + { + audioGraph.Stop(); + + }; + } + + + + private MediaSource mediaSource; + public async Task CreateMediaSourceInputNode2(Uri contentUri) + { + if (audioGraph == null) + return null; + + mediaSource = MediaSource.CreateFromUri(contentUri); + CreateMediaSourceAudioInputNodeResult mediaSourceAudioInputNodeResult = + await audioGraph.CreateMediaSourceAudioInputNodeAsync(mediaSource); + + if (mediaSourceAudioInputNodeResult.Status != MediaSourceAudioInputNodeCreationStatus.Success) + { + switch (mediaSourceAudioInputNodeResult.Status) + { + case MediaSourceAudioInputNodeCreationStatus.FormatNotSupported: + Debug.WriteLine("The MediaSource uses an unsupported format"); + break; + case MediaSourceAudioInputNodeCreationStatus.NetworkError: + Debug.WriteLine("The MediaSource requires a network connection and a network-related error occurred"); + break; + case MediaSourceAudioInputNodeCreationStatus.UnknownFailure: + default: + Debug.WriteLine("An unknown error occurred while opening the MediaSource"); + break; + } + return null; + } + + return mediaSourceAudioInputNodeResult.Node; + + + //mediaSourceInputNode.MediaSourceCompleted += (s, e) => { + // audioGraph.Stop(); + + //}; + } + + + + private async Task CreateFileOutputNode() + { + FileSavePicker saveFilePicker = new FileSavePicker(); + saveFilePicker.FileTypeChoices.Add("Pulse Code Modulation", new List() { ".wav" }); + saveFilePicker.FileTypeChoices.Add("Windows Media Audio", new List() { ".wma" }); + saveFilePicker.FileTypeChoices.Add("MPEG Audio Layer-3", new List() { ".mp3" }); + saveFilePicker.SuggestedFileName = "New Audio Track"; + StorageFile file = await saveFilePicker.PickSaveFileAsync(); + + // File can be null if cancel is hit in the file picker + if (file == null) + { + return; + } + + Windows.Media.MediaProperties.MediaEncodingProfile mediaEncodingProfile; + switch (file.FileType.ToString().ToLowerInvariant()) + { + case ".wma": + mediaEncodingProfile = MediaEncodingProfile.CreateWma(AudioEncodingQuality.High); + break; + case ".mp3": + mediaEncodingProfile = MediaEncodingProfile.CreateMp3(AudioEncodingQuality.High); + break; + case ".wav": + mediaEncodingProfile = MediaEncodingProfile.CreateWav(AudioEncodingQuality.High); + break; + default: + throw new ArgumentException(); + } + + + // Operate node at the graph format, but save file at the specified format + CreateAudioFileOutputNodeResult result = await audioGraph.CreateFileOutputNodeAsync(file, mediaEncodingProfile); + + if (result.Status != AudioFileNodeCreationStatus.Success) + { + // FileOutputNode creation failed + //ShowErrorMessage(result.Status.ToString()); + return; + } + + //fileOutputNode = result.FileOutputNode; + } + + + + + + public AudioFrameOutputNode frameOutputNode; + + public AudioFrameInputNode frameInputNode; + + private void CreateFrameInputNode() + { + // Create the FrameInputNode at the same format as the graph, except explicitly set mono. + AudioEncodingProperties nodeEncodingProperties = audioGraph.EncodingProperties; + nodeEncodingProperties.ChannelCount = 1; + frameInputNode = audioGraph.CreateFrameInputNode(nodeEncodingProperties); + + // Initialize the Frame Input Node in the stopped state + frameInputNode.Stop(); + + // Hook up an event handler so we can start generating samples when needed + // This event is triggered when the node is required to provide data + frameInputNode.QuantumStarted += (s, args) => + { + + uint numSamplesNeeded = (uint)args.RequiredSamples; + + if (numSamplesNeeded != 0) + { + //AudioFrame audioData = GenerateAudioData(numSamplesNeeded); + //frameInputNode.AddFrame(audioData); + } + }; + } + + + + } + +} diff --git a/CaptureEncoder/CaptureEncoder.csproj b/CaptureEncoder/CaptureEncoder.csproj index b9450c4..4aacc4b 100644 --- a/CaptureEncoder/CaptureEncoder.csproj +++ b/CaptureEncoder/CaptureEncoder.csproj @@ -11,7 +11,7 @@ CaptureEncoder en-US UAP - 10.0.18362.0 + 10.0.19041.0 10.0.17763.0 14 512 @@ -121,6 +121,7 @@ PackageReference + diff --git a/CaptureEncoder/Encoder.cs b/CaptureEncoder/Encoder.cs index f8074e9..128c2fd 100644 --- a/CaptureEncoder/Encoder.cs +++ b/CaptureEncoder/Encoder.cs @@ -53,6 +53,17 @@ private async Task EncodeInternalAsync(IRandomAccessStream stream, uint width, u encodingProfile.Video.FrameRate.Denominator = 1; encodingProfile.Video.PixelAspectRatio.Numerator = 1; encodingProfile.Video.PixelAspectRatio.Denominator = 1; + + + encodingProfile.Audio.SampleRate = 960; + Debug.WriteLine("01"); + encodingProfile.Audio.Subtype = "AAC"; + encodingProfile.Audio.Bitrate = 16; + encodingProfile.Audio.ChannelCount = 2; + Debug.WriteLine("02"); + + + var transcode = await _transcoder.PrepareMediaStreamSourceTranscodeAsync(_mediaStreamSource, stream, encodingProfile); await transcode.TranscodeAsync(); @@ -90,9 +101,11 @@ private void CreateMediaObjects() // Describe our input: uncompressed BGRA8 buffers var videoProperties = VideoEncodingProperties.CreateUncompressed(MediaEncodingSubtypes.Bgra8, (uint)width, (uint)height); _videoDescriptor = new VideoStreamDescriptor(videoProperties); - + // Describe audio input + var audioProperties = AudioEncodingProperties.CreateMp3(960, 2, 16); + _audioDescriptor = new AudioStreamDescriptor(audioProperties); // Create our MediaStreamSource - _mediaStreamSource = new MediaStreamSource(_videoDescriptor); + _mediaStreamSource = new MediaStreamSource(_videoDescriptor, _audioDescriptor); _mediaStreamSource.BufferTime = TimeSpan.FromSeconds(0); _mediaStreamSource.Starting += OnMediaStreamSourceStarting; _mediaStreamSource.SampleRequested += OnMediaStreamSourceSampleRequested; @@ -108,20 +121,36 @@ private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaS { try { - using (var frame = _frameGenerator.WaitForNewFrame()) + + if (args.Request.StreamDescriptor.GetType() == typeof(VideoStreamDescriptor)) { - if (frame == null) + //request video + using (var frame = _frameGenerator.WaitForNewFrame()) { - args.Request.Sample = null; - DisposeInternal(); - return; + if (frame == null) + { + args.Request.Sample = null; + DisposeInternal(); + return; + } + + var timeStamp = frame.SystemRelativeTime; + + var sample = MediaStreamSample.CreateFromDirect3D11Surface(frame.Surface, timeStamp); + args.Request.Sample = sample; } + } + else if (args.Request.StreamDescriptor.GetType() == typeof(AudioStreamDescriptor)) + { + //request audio - var timeStamp = frame.SystemRelativeTime; - var sample = MediaStreamSample.CreateFromDirect3D11Surface(frame.Surface, timeStamp); - args.Request.Sample = sample; } + + + + + } catch (Exception e) { @@ -153,6 +182,7 @@ private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSo private CaptureFrameWait _frameGenerator; private VideoStreamDescriptor _videoDescriptor; + private AudioStreamDescriptor _audioDescriptor; private MediaStreamSource _mediaStreamSource; private MediaTranscoder _transcoder; private bool _isRecording; diff --git a/SimpleRecorder/SimpleRecorder.csproj b/SimpleRecorder/SimpleRecorder.csproj index aebe0a5..60fa6c0 100644 --- a/SimpleRecorder/SimpleRecorder.csproj +++ b/SimpleRecorder/SimpleRecorder.csproj @@ -11,7 +11,7 @@ SimpleRecorder en-US UAP - 10.0.18362.0 + 10.0.19041.0 10.0.17763.0 14 512 From 6ca3841220c8dbc9bc5fc82601a84fe122af86ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=8B=E4=BB=95=E9=92=8A?= Date: Tue, 22 Jun 2021 02:32:27 +0800 Subject: [PATCH 2/4] add audio support --- CaptureEncoder/Audio.cs | 11 +++++- CaptureEncoder/CaptureEncoder.csproj | 1 + CaptureEncoder/Encoder.cs | 55 ++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/CaptureEncoder/Audio.cs b/CaptureEncoder/Audio.cs index ba3ce10..a10416a 100644 --- a/CaptureEncoder/Audio.cs +++ b/CaptureEncoder/Audio.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; using Windows.Media.Audio; @@ -61,7 +62,7 @@ public async Task InitAudioGraph() AudioDeviceInputNode deviceInputNode; - private async Task CreateDeviceInputNode() + public async Task CreateDeviceInputNode() { // Create a device output node CreateAudioDeviceInputNodeResult result = await audioGraph.CreateDeviceInputNodeAsync(Windows.Media.Capture.MediaCategory.Media); @@ -306,4 +307,12 @@ private void CreateFrameInputNode() } + [ComImport] + [Guid("5B0D3235-4DBA-4D44-865E-8F1D0E4FD04D")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + unsafe interface IMemoryBufferByteAccess + { + void GetBuffer(out byte* buffer, out uint capacity); + } + } diff --git a/CaptureEncoder/CaptureEncoder.csproj b/CaptureEncoder/CaptureEncoder.csproj index 4aacc4b..9964aca 100644 --- a/CaptureEncoder/CaptureEncoder.csproj +++ b/CaptureEncoder/CaptureEncoder.csproj @@ -27,6 +27,7 @@ DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP prompt 4 + true AnyCPU diff --git a/CaptureEncoder/Encoder.cs b/CaptureEncoder/Encoder.cs index 128c2fd..344ef80 100644 --- a/CaptureEncoder/Encoder.cs +++ b/CaptureEncoder/Encoder.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Runtime.InteropServices; using System.Threading.Tasks; using Windows.Foundation; using Windows.Graphics.Capture; @@ -115,7 +116,7 @@ private void CreateMediaObjects() _transcoder.HardwareAccelerationEnabled = true; } - private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaStreamSourceSampleRequestedEventArgs args) + unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaStreamSourceSampleRequestedEventArgs args) { if (_isRecording && !_closed) { @@ -124,7 +125,7 @@ private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaS if (args.Request.StreamDescriptor.GetType() == typeof(VideoStreamDescriptor)) { - //request video + // Request Video using (var frame = _frameGenerator.WaitForNewFrame()) { if (frame == null) @@ -142,9 +143,26 @@ private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaS } else if (args.Request.StreamDescriptor.GetType() == typeof(AudioStreamDescriptor)) { - //request audio + // Request Audio + MyAudioGraphPlayer player = new MyAudioGraphPlayer(); + var frame = player.frameOutputNode.GetFrame(); + var audioBuffer = frame.LockBuffer(Windows.Media.AudioBufferAccessMode.Read); + IMemoryBufferReference bufferReference = audioBuffer.CreateReference(); + ((IMemoryBufferByteAccess)bufferReference).GetBuffer(out byte* dataInBytes, out uint capacityInBytes); + + using (var dataWriter = new DataWriter()) + { + byte[] write = new byte[capacityInBytes]; + Marshal.Copy((IntPtr)dataInBytes, write, 0, write.Length); + dataWriter.WriteBytes(write); + + var x = MediaStreamSample.CreateFromBuffer(dataWriter.DetachBuffer(), frame.SystemRelativeTime.Value); + args.Request.Sample= x; + } + + // } @@ -168,6 +186,37 @@ private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaS } } + + unsafe MediaStreamSample GetAudioSample() + { + + + //request audio + MyAudioGraphPlayer player = new MyAudioGraphPlayer(); + var frame = player.frameOutputNode.GetFrame(); + var audioBuffer = frame.LockBuffer(Windows.Media.AudioBufferAccessMode.Read); + + IMemoryBufferReference bufferReference = audioBuffer.CreateReference(); + + ((IMemoryBufferByteAccess)bufferReference).GetBuffer(out byte* dataInBytes, out uint capacityInBytes); + + using (var dataWriter = new DataWriter()) + { + byte[] write = new byte[capacityInBytes]; + Marshal.Copy((IntPtr)dataInBytes, write, 0, write.Length); + dataWriter.WriteBytes(write); + + var x = MediaStreamSample.CreateFromBuffer(dataWriter.DetachBuffer(), frame.SystemRelativeTime.Value); + return x; + } + + + + + } + + + private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSourceStartingEventArgs args) { using (var frame = _frameGenerator.WaitForNewFrame()) From 15d1bb8f9bb3fcad4bcd9bcf0ba324897af43041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=8B=E4=BB=95=E9=92=8A?= Date: Fri, 25 Jun 2021 07:37:06 +0800 Subject: [PATCH 3/4] fix --- CaptureEncoder/AppSettingHelper.cs | 76 +++++++ CaptureEncoder/Audio.cs | 318 --------------------------- CaptureEncoder/AudioInterface.cs | 25 +++ CaptureEncoder/CaptureEncoder.csproj | 6 +- CaptureEncoder/Encoder.cs | 271 ++++++++++++++++++----- SimpleRecorder/Package.appxmanifest | 2 + SimpleRecorder/SimpleRecorder.csproj | 2 +- 7 files changed, 327 insertions(+), 373 deletions(-) create mode 100644 CaptureEncoder/AppSettingHelper.cs delete mode 100644 CaptureEncoder/Audio.cs create mode 100644 CaptureEncoder/AudioInterface.cs diff --git a/CaptureEncoder/AppSettingHelper.cs b/CaptureEncoder/AppSettingHelper.cs new file mode 100644 index 0000000..b5a8204 --- /dev/null +++ b/CaptureEncoder/AppSettingHelper.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Windows.Storage; + +namespace CaptureEncoder +{ + //public static class AppSettingHelper + //{ + // private static ApplicationDataContainer root = ApplicationData.Current.LocalSettings; + + // public static string key_RecordMike = "IsMikeRecord"; + // public static string key_InputDeviceIndex = "MikeRecordedIndex"; + // public static string key_OutputDeviceRecordedIndex = "OutputDeviceRecordedIndex"; + + // public static bool UseMikeAsInput + // { + // get + // { + // return ReadSetting(key_RecordMike); + + // } + // } + + // public static int InputDeviceIndex + // { + // get + // { + // return ReadSetting(key_InputDeviceIndex); + + // } + // } + + // public static int OutputDeviceRecordedIndex + // { + // get + // { + // return ReadSetting(key_OutputDeviceRecordedIndex); + + // } + // } + + + + // // + // public static T ReadSetting(string key) + // { + // if (root.Values.TryGetValue(key, out object value)) + // { + // return (T)value; + // } + // else + // { + // return default(T); + // } + // } + + // // + // public static void WriteSetting(string key, T Tvalue)// where T : struct + // { + // ApplicationDataContainer root = ApplicationData.Current.LocalSettings; + // if (root.Values.TryGetValue(key, out object oldkey)) + // { + + // root.Values[key] = Tvalue; + // } + // else + // { + // root.Values.Add(key, Tvalue); + // } + // } + + //} +} diff --git a/CaptureEncoder/Audio.cs b/CaptureEncoder/Audio.cs deleted file mode 100644 index a10416a..0000000 --- a/CaptureEncoder/Audio.cs +++ /dev/null @@ -1,318 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading.Tasks; -using Windows.Media.Audio; -using Windows.Media.Core; -using Windows.Media.MediaProperties; -using Windows.Media.Streaming.Adaptive; -using Windows.Storage; -using Windows.Storage.Pickers; - -namespace CaptureEncoder -{ - public sealed class MyAudioGraphPlayer - { - - public AudioGraph audioGraph; - public MyAudioGraphPlayer() - { - - - - } - - - - - - /// - /// 初始化音频图 - /// - /// - public async Task InitAudioGraph() - { - AudioGraphSettings settings = new AudioGraphSettings(Windows.Media.Render.AudioRenderCategory.Media); - CreateAudioGraphResult result = await AudioGraph.CreateAsync(settings); - if (result.Status != AudioGraphCreationStatus.Success) - { - Debug.WriteLine("AudioGraph creation error: " + result.Status.ToString()); - } - audioGraph = result.Graph; - audioGraph.UnrecoverableErrorOccurred += async (sender, args) => - { - if (sender == audioGraph && args.Error != AudioGraphUnrecoverableError.None) - { - Debug.WriteLine("The audio graph encountered and unrecoverable error."); - audioGraph.Stop(); - audioGraph.Dispose(); - await InitAudioGraph(); - } - }; - } - - - - - - #region 收起 - - AudioDeviceInputNode deviceInputNode; - - public async Task CreateDeviceInputNode() - { - // Create a device output node - CreateAudioDeviceInputNodeResult result = await audioGraph.CreateDeviceInputNodeAsync(Windows.Media.Capture.MediaCategory.Media); - - if (result.Status != AudioDeviceNodeCreationStatus.Success) - { - // Cannot create device output node - //ShowErrorMessage(result.Status.ToString()); - return; - } - - deviceInputNode = result.DeviceInputNode; - } - - - - #endregion - - - - - public AudioDeviceOutputNode deviceOutputNode; - public async Task CreateDeviceOutputNode() - { - // Create a device output node - CreateAudioDeviceOutputNodeResult result = await audioGraph.CreateDeviceOutputNodeAsync(); - - if (result.Status != AudioDeviceNodeCreationStatus.Success) - { - // Cannot create device output node - //ShowErrorMessage(result.Status.ToString()); - return null; - } - - return result.DeviceOutputNode; - } - - - private async Task CreateFileInputNode() - { - if (audioGraph == null) - return null; - - FileOpenPicker filePicker = new FileOpenPicker(); - filePicker.SuggestedStartLocation = PickerLocationId.MusicLibrary; - filePicker.FileTypeFilter.Add(".mp3"); - filePicker.FileTypeFilter.Add(".wav"); - filePicker.FileTypeFilter.Add(".wma"); - filePicker.FileTypeFilter.Add(".m4a"); - filePicker.ViewMode = PickerViewMode.Thumbnail; - StorageFile file = await filePicker.PickSingleFileAsync(); - - // File can be null if cancel is hit in the file picker - if (file == null) - { - return null; - } - CreateAudioFileInputNodeResult result = await audioGraph.CreateFileInputNodeAsync(file); - - if (result.Status != AudioFileNodeCreationStatus.Success) - { - //ShowErrorMessage(result.Status.ToString()); - } - - return result.FileInputNode; - } - - - - - - - public MediaSourceAudioInputNode mediaSourceInputNode; - - public async Task CreateMediaSourceInputNode(System.Uri contentUri) - { - if (audioGraph == null) - return; - - var adaptiveMediaSourceResult = await AdaptiveMediaSource.CreateFromUriAsync(contentUri); - if (adaptiveMediaSourceResult.Status != AdaptiveMediaSourceCreationStatus.Success) - { - Debug.WriteLine("Failed to create AdaptiveMediaSource"); - - - - return; - } - - var mediaSource = MediaSource.CreateFromAdaptiveMediaSource(adaptiveMediaSourceResult.MediaSource); - CreateMediaSourceAudioInputNodeResult mediaSourceAudioInputNodeResult = - await audioGraph.CreateMediaSourceAudioInputNodeAsync(mediaSource); - - if (mediaSourceAudioInputNodeResult.Status != MediaSourceAudioInputNodeCreationStatus.Success) - { - switch (mediaSourceAudioInputNodeResult.Status) - { - case MediaSourceAudioInputNodeCreationStatus.FormatNotSupported: - Debug.WriteLine("The MediaSource uses an unsupported format"); - break; - case MediaSourceAudioInputNodeCreationStatus.NetworkError: - Debug.WriteLine("The MediaSource requires a network connection and a network-related error occurred"); - break; - case MediaSourceAudioInputNodeCreationStatus.UnknownFailure: - default: - Debug.WriteLine("An unknown error occurred while opening the MediaSource"); - break; - } - return; - } - - mediaSourceInputNode = mediaSourceAudioInputNodeResult.Node; - - - mediaSourceInputNode.MediaSourceCompleted += (s, e) => - { - audioGraph.Stop(); - - }; - } - - - - private MediaSource mediaSource; - public async Task CreateMediaSourceInputNode2(Uri contentUri) - { - if (audioGraph == null) - return null; - - mediaSource = MediaSource.CreateFromUri(contentUri); - CreateMediaSourceAudioInputNodeResult mediaSourceAudioInputNodeResult = - await audioGraph.CreateMediaSourceAudioInputNodeAsync(mediaSource); - - if (mediaSourceAudioInputNodeResult.Status != MediaSourceAudioInputNodeCreationStatus.Success) - { - switch (mediaSourceAudioInputNodeResult.Status) - { - case MediaSourceAudioInputNodeCreationStatus.FormatNotSupported: - Debug.WriteLine("The MediaSource uses an unsupported format"); - break; - case MediaSourceAudioInputNodeCreationStatus.NetworkError: - Debug.WriteLine("The MediaSource requires a network connection and a network-related error occurred"); - break; - case MediaSourceAudioInputNodeCreationStatus.UnknownFailure: - default: - Debug.WriteLine("An unknown error occurred while opening the MediaSource"); - break; - } - return null; - } - - return mediaSourceAudioInputNodeResult.Node; - - - //mediaSourceInputNode.MediaSourceCompleted += (s, e) => { - // audioGraph.Stop(); - - //}; - } - - - - private async Task CreateFileOutputNode() - { - FileSavePicker saveFilePicker = new FileSavePicker(); - saveFilePicker.FileTypeChoices.Add("Pulse Code Modulation", new List() { ".wav" }); - saveFilePicker.FileTypeChoices.Add("Windows Media Audio", new List() { ".wma" }); - saveFilePicker.FileTypeChoices.Add("MPEG Audio Layer-3", new List() { ".mp3" }); - saveFilePicker.SuggestedFileName = "New Audio Track"; - StorageFile file = await saveFilePicker.PickSaveFileAsync(); - - // File can be null if cancel is hit in the file picker - if (file == null) - { - return; - } - - Windows.Media.MediaProperties.MediaEncodingProfile mediaEncodingProfile; - switch (file.FileType.ToString().ToLowerInvariant()) - { - case ".wma": - mediaEncodingProfile = MediaEncodingProfile.CreateWma(AudioEncodingQuality.High); - break; - case ".mp3": - mediaEncodingProfile = MediaEncodingProfile.CreateMp3(AudioEncodingQuality.High); - break; - case ".wav": - mediaEncodingProfile = MediaEncodingProfile.CreateWav(AudioEncodingQuality.High); - break; - default: - throw new ArgumentException(); - } - - - // Operate node at the graph format, but save file at the specified format - CreateAudioFileOutputNodeResult result = await audioGraph.CreateFileOutputNodeAsync(file, mediaEncodingProfile); - - if (result.Status != AudioFileNodeCreationStatus.Success) - { - // FileOutputNode creation failed - //ShowErrorMessage(result.Status.ToString()); - return; - } - - //fileOutputNode = result.FileOutputNode; - } - - - - - - public AudioFrameOutputNode frameOutputNode; - - public AudioFrameInputNode frameInputNode; - - private void CreateFrameInputNode() - { - // Create the FrameInputNode at the same format as the graph, except explicitly set mono. - AudioEncodingProperties nodeEncodingProperties = audioGraph.EncodingProperties; - nodeEncodingProperties.ChannelCount = 1; - frameInputNode = audioGraph.CreateFrameInputNode(nodeEncodingProperties); - - // Initialize the Frame Input Node in the stopped state - frameInputNode.Stop(); - - // Hook up an event handler so we can start generating samples when needed - // This event is triggered when the node is required to provide data - frameInputNode.QuantumStarted += (s, args) => - { - - uint numSamplesNeeded = (uint)args.RequiredSamples; - - if (numSamplesNeeded != 0) - { - //AudioFrame audioData = GenerateAudioData(numSamplesNeeded); - //frameInputNode.AddFrame(audioData); - } - }; - } - - - - } - - [ComImport] - [Guid("5B0D3235-4DBA-4D44-865E-8F1D0E4FD04D")] - [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] - unsafe interface IMemoryBufferByteAccess - { - void GetBuffer(out byte* buffer, out uint capacity); - } - -} diff --git a/CaptureEncoder/AudioInterface.cs b/CaptureEncoder/AudioInterface.cs new file mode 100644 index 0000000..359bb28 --- /dev/null +++ b/CaptureEncoder/AudioInterface.cs @@ -0,0 +1,25 @@ +using System; +using System.Runtime.InteropServices; + +namespace CaptureEncoder +{ + + // To populate an AudioFrame with audio data, + // you must get access to the underlying memory buffer + // of the audio frame.To do this you must initialize + // the IMemoryBufferByteAccess COM interface + // by adding the following code within your namespace. + // + [ComImport] + [Guid("5B0D3235-4DBA-4D44-865E-8F1D0E4FD04D")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + unsafe interface IMemoryBufferByteAccess + { + void GetBuffer(out byte* buffer, out uint capacity); + } + // You must also configure your project in Microsoft Visual Studio + // to allow the compilation of unsafe code + // by opening the project's Properties page, + // clicking the Build property page, + // and selecting the Allow Unsafe Code checkbox +} diff --git a/CaptureEncoder/CaptureEncoder.csproj b/CaptureEncoder/CaptureEncoder.csproj index 9964aca..c42b0c2 100644 --- a/CaptureEncoder/CaptureEncoder.csproj +++ b/CaptureEncoder/CaptureEncoder.csproj @@ -11,7 +11,7 @@ CaptureEncoder en-US UAP - 10.0.19041.0 + 10.0.17763.0 10.0.17763.0 14 512 @@ -107,6 +107,7 @@ full false prompt + true x64 @@ -122,7 +123,8 @@ PackageReference - + + diff --git a/CaptureEncoder/Encoder.cs b/CaptureEncoder/Encoder.cs index 344ef80..aacdc8e 100644 --- a/CaptureEncoder/Encoder.cs +++ b/CaptureEncoder/Encoder.cs @@ -4,13 +4,20 @@ using System; using System.Diagnostics; using System.Runtime.InteropServices; +using System.Runtime.InteropServices.WindowsRuntime; using System.Threading.Tasks; +using Windows.Devices.Enumeration; using Windows.Foundation; using Windows.Graphics.Capture; using Windows.Graphics.DirectX.Direct3D11; +using Windows.Media; +using Windows.Media.Audio; +using Windows.Media.Capture; using Windows.Media.Core; +using Windows.Media.Devices; using Windows.Media.MediaProperties; using Windows.Media.Transcoding; +using Windows.Storage; using Windows.Storage.Streams; namespace CaptureEncoder @@ -24,8 +31,80 @@ public Encoder(IDirect3DDevice device, GraphicsCaptureItem item) _isRecording = false; CreateMediaObjects(); + + } + private async Task CreateAudioObjects() + { + // create AudioGraph + AudioGraphSettings settings = new AudioGraphSettings(Windows.Media.Render.AudioRenderCategory.Media); + settings.QuantumSizeSelectionMode = QuantumSizeSelectionMode.LowestLatency; + var outputDevices = await DeviceInformation.FindAllAsync(MediaDevice.GetAudioRenderSelector()); + settings.PrimaryRenderDevice = outputDevices[0]; + var result = await AudioGraph.CreateAsync(settings); + if (result.Status != AudioGraphCreationStatus.Success) + { + Debug.WriteLine("AudioGraph creation error: " + result.Status.ToString()); + return; + } + _audioGraph = result.Graph; + + _audioGraph.UnrecoverableErrorOccurred += (sender,e) => { + sender.Dispose(); + }; + //_audioGraph.QuantumStarted += _audioGraph_QuantumStarted; + + // create device output + var deviceOutputResult = await _audioGraph.CreateDeviceOutputNodeAsync(); + if (deviceOutputResult.Status != AudioDeviceNodeCreationStatus.Success) + { + Debug.WriteLine("Cannot create device output node"); + return; + } + _deviceOutputNode= deviceOutputResult.DeviceOutputNode; + + // create frame output + _frameOutputNode = _audioGraph.CreateFrameOutputNode(); + //_frameOutputNode.ConsumeInput = true; + + //_frameOutputNode.Start(); + // create device input + var deviceInputResult = await _audioGraph.CreateDeviceInputNodeAsync(MediaCategory.Other); + if (deviceInputResult.Status != AudioDeviceNodeCreationStatus.Success) + { + Debug.WriteLine($"Audio Device Input unavailable because {deviceInputResult.Status.ToString()}"); + + return; + } + _deviceInputNode = deviceInputResult.DeviceInputNode; + + _deviceInputNode.AddOutgoingConnection(_deviceOutputNode); + //_deviceInputNode.AddOutgoingConnection(_frameOutputNode); + + //mp3File = await KnownFolders.VideosLibrary.CreateFileAsync("temp.mp3", CreationCollisionOption.ReplaceExisting); + //var fileOutputNodeResult=await _audioGraph.CreateFileOutputNodeAsync(mp3File, MediaEncodingProfile.CreateMp3(AudioEncodingQuality.High)); + //if (fileOutputNodeResult.Status != AudioFileNodeCreationStatus.Success) + //{ + // Debug.WriteLine($"Audio File output unavailable because {deviceInputResult.Status.ToString()}"); + + // return; + //} + + //_fileOutputNode = fileOutputNodeResult.FileOutputNode; + + + // + + + + + + //_deviceInputNode.AddOutgoingConnection(_fileOutputNode); + + } + + public IAsyncAction EncodeAsync(IRandomAccessStream stream, uint width, uint height, uint bitrateInBps, uint frameRate) { return EncodeInternalAsync(stream, width, height, bitrateInBps, frameRate).AsAsyncAction(); @@ -45,8 +124,8 @@ private async Task EncodeInternalAsync(IRandomAccessStream stream, uint width, u using (_frameGenerator) { var encodingProfile = new MediaEncodingProfile(); - encodingProfile.Container.Subtype = "MPEG4"; - encodingProfile.Video.Subtype = "H264"; + encodingProfile.Container.Subtype = MediaEncodingSubtypes.Mpeg4; + encodingProfile.Video.Subtype = MediaEncodingSubtypes.H264; encodingProfile.Video.Width = width; encodingProfile.Video.Height = height; encodingProfile.Video.Bitrate = bitrateInBps; @@ -56,14 +135,14 @@ private async Task EncodeInternalAsync(IRandomAccessStream stream, uint width, u encodingProfile.Video.PixelAspectRatio.Denominator = 1; - encodingProfile.Audio.SampleRate = 960; - Debug.WriteLine("01"); - encodingProfile.Audio.Subtype = "AAC"; - encodingProfile.Audio.Bitrate = 16; - encodingProfile.Audio.ChannelCount = 2; - Debug.WriteLine("02"); + // Describe audio input + await CreateAudioObjects(); + _audioDescriptor = new AudioStreamDescriptor(_audioGraph.EncodingProperties); + _mediaStreamSource.AddStreamDescriptor(_audioDescriptor); + encodingProfile.Audio = MediaEncodingProfile.CreateFlac(AudioEncodingQuality.Low).Audio; + var transcode = await _transcoder.PrepareMediaStreamSourceTranscodeAsync(_mediaStreamSource, stream, encodingProfile); @@ -88,13 +167,17 @@ public void Dispose() _isRecording = false; } - private void DisposeInternal() + private void DisposeInternal() { _frameGenerator.Dispose(); + } private void CreateMediaObjects() { + + + // Create our encoding profile based on the size of the item int width = _captureItem.Size.Width; int height = _captureItem.Size.Height; @@ -102,20 +185,29 @@ private void CreateMediaObjects() // Describe our input: uncompressed BGRA8 buffers var videoProperties = VideoEncodingProperties.CreateUncompressed(MediaEncodingSubtypes.Bgra8, (uint)width, (uint)height); _videoDescriptor = new VideoStreamDescriptor(videoProperties); - // Describe audio input - var audioProperties = AudioEncodingProperties.CreateMp3(960, 2, 16); - _audioDescriptor = new AudioStreamDescriptor(audioProperties); + + // audio + + // Create our MediaStreamSource - _mediaStreamSource = new MediaStreamSource(_videoDescriptor, _audioDescriptor); + _mediaStreamSource = new MediaStreamSource(_videoDescriptor); _mediaStreamSource.BufferTime = TimeSpan.FromSeconds(0); _mediaStreamSource.Starting += OnMediaStreamSourceStarting; _mediaStreamSource.SampleRequested += OnMediaStreamSourceSampleRequested; + _mediaStreamSource.Closed += (s,e) => { + Debug.WriteLine("Stop AudioGraph"); + _audioGraph?.Stop(); + + }; + + // Create our transcoder _transcoder = new MediaTranscoder(); _transcoder.HardwareAccelerationEnabled = true; } + TimeSpan delay=new TimeSpan(0); unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaStreamSourceSampleRequestedEventArgs args) { if (_isRecording && !_closed) @@ -126,6 +218,7 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, if (args.Request.StreamDescriptor.GetType() == typeof(VideoStreamDescriptor)) { // Request Video + using (var frame = _frameGenerator.WaitForNewFrame()) { if (frame == null) @@ -136,39 +229,56 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, } var timeStamp = frame.SystemRelativeTime; - + //Debug.WriteLine($"video:{timeStamp.TotalMilliseconds}"); + //_time = timeStamp; var sample = MediaStreamSample.CreateFromDirect3D11Surface(frame.Surface, timeStamp); args.Request.Sample = sample; } } else if (args.Request.StreamDescriptor.GetType() == typeof(AudioStreamDescriptor)) { - // Request Audio - MyAudioGraphPlayer player = new MyAudioGraphPlayer(); - var frame = player.frameOutputNode.GetFrame(); - var audioBuffer = frame.LockBuffer(Windows.Media.AudioBufferAccessMode.Read); + var request = args.Request; - IMemoryBufferReference bufferReference = audioBuffer.CreateReference(); + var deferal = request.GetDeferral(); - ((IMemoryBufferByteAccess)bufferReference).GetBuffer(out byte* dataInBytes, out uint capacityInBytes); + var frame = GetFrameAsync(); - using (var dataWriter = new DataWriter()) + + using (AudioBuffer buffer = frame.LockBuffer(AudioBufferAccessMode.Write)) + using (IMemoryBufferReference reference = buffer.CreateReference()) { - byte[] write = new byte[capacityInBytes]; - Marshal.Copy((IntPtr)dataInBytes, write, 0, write.Length); - dataWriter.WriteBytes(write); + byte* dataInBytes; + uint capacityInBytes; + // Get the buffer from the AudioFrame + ((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacityInBytes); + byte[] bytes = new byte[capacityInBytes]; + Marshal.Copy((IntPtr)dataInBytes, bytes, 0, (int)capacityInBytes); + var data_buffer = WindowsRuntimeBufferExtensions.AsBuffer(bytes, 0, (int)capacityInBytes); + + var stamp = time_start + frame.RelativeTime.GetValueOrDefault(); + var duration = frame.Duration.GetValueOrDefault(); + + var sample = MediaStreamSample.CreateFromBuffer(data_buffer, stamp); + sample.Duration = duration;// frame.Duration.GetValueOrDefault(); + sample.KeyFrame = true; + + if (sample.Discontinuous) + { + Debug.WriteLine("lost sample"); + sample.Discontinuous = false; + } + //Debug.WriteLine($"audio:{stamp.TotalMilliseconds}duration:{sample.Duration.TotalMilliseconds}"); - var x = MediaStreamSample.CreateFromBuffer(dataWriter.DetachBuffer(), frame.SystemRelativeTime.Value); - args.Request.Sample= x; + request.Sample = sample; + //Debug.WriteLine($"bytesize:{capacityInBytes}time:{frame.Duration.GetValueOrDefault().TotalMilliseconds}"); } - - // - } + + deferal.Complete(); + } - } catch (Exception e) { @@ -187,42 +297,58 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, } - unsafe MediaStreamSample GetAudioSample() - { + AudioFrame GetFrameAsync() { + var frame = _frameOutputNode.GetFrame(); + if (frame.Duration.GetValueOrDefault().TotalSeconds != 0) + { + return frame; - //request audio - MyAudioGraphPlayer player = new MyAudioGraphPlayer(); - var frame = player.frameOutputNode.GetFrame(); - var audioBuffer = frame.LockBuffer(Windows.Media.AudioBufferAccessMode.Read); + } + else + { + Debug.Write("Delay"); + Task.Delay(5); + return GetFrameAsync(); + } + } - IMemoryBufferReference bufferReference = audioBuffer.CreateReference(); - ((IMemoryBufferByteAccess)bufferReference).GetBuffer(out byte* dataInBytes, out uint capacityInBytes); - using (var dataWriter = new DataWriter()) - { - byte[] write = new byte[capacityInBytes]; - Marshal.Copy((IntPtr)dataInBytes, write, 0, write.Length); - dataWriter.WriteBytes(write); - var x = MediaStreamSample.CreateFromBuffer(dataWriter.DetachBuffer(), frame.SystemRelativeTime.Value); - return x; + TimeSpan time_start=new TimeSpan(); + private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSourceStartingEventArgs args) + { + MediaStreamSourceStartingRequest request = args.Request; + + + using (var frame = _frameGenerator.WaitForNewFrame()) + { + time_start = frame.SystemRelativeTime; + request.SetActualStartPosition(frame.SystemRelativeTime); + + } + _audioGraph?.Start(); + using (var audioFrame = _frameOutputNode.GetFrame()) + { + time_start= time_start-audioFrame.RelativeTime.GetValueOrDefault(); } + //if ((request.StartPosition != null)) + //{ + // UInt64 sampleOffset = (UInt64)request.StartPosition.Value.Ticks / (UInt64)sampleDuration.Ticks; + // timeOffset = new TimeSpan((long)sampleOffset * sampleDuration.Ticks); + // byteOffset = sampleOffset * sampleSize; + // Debug.WriteLine($"timeOffset:{timeOffset.TotalMilliseconds}ms"); + //} + + - } - private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSourceStartingEventArgs args) - { - using (var frame = _frameGenerator.WaitForNewFrame()) - { - args.Request.SetActualStartPosition(frame.SystemRelativeTime); - } } private IDirect3DDevice _device; @@ -236,5 +362,46 @@ private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSo private MediaTranscoder _transcoder; private bool _isRecording; private bool _closed = false; + + // audio graph and nodes + private AudioGraph _audioGraph; + private AudioDeviceInputNode _deviceInputNode; + private AudioFrameOutputNode _frameOutputNode; + private AudioDeviceOutputNode _deviceOutputNode; + private AudioFileOutputNode _fileOutputNode; + private const UInt32 sampleSize = 960; + private TimeSpan sampleDuration = TimeSpan.FromMilliseconds(10); + private InMemoryRandomAccessStream _memoryStream = new InMemoryRandomAccessStream(); + private IRandomAccessStream _fileStream; + + unsafe private void _audioGraph_QuantumStarted(AudioGraph sender, object args) + { + AudioFrame frame = _frameOutputNode?.GetFrame(); + + using (AudioBuffer buffer = frame.LockBuffer(AudioBufferAccessMode.Write)) + using (IMemoryBufferReference reference = buffer.CreateReference()) + { + byte* dataInBytes; + uint capacityInBytes; + // Get the buffer from the AudioFrame + ((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacityInBytes); + byte[] bytes = new byte[capacityInBytes]; + Marshal.Copy((IntPtr)dataInBytes, bytes, 0, (int)capacityInBytes); + var data_buffer = WindowsRuntimeBufferExtensions.AsBuffer(bytes, 0, (int)capacityInBytes); + + WriteIntoMemory(data_buffer); + //Debug.WriteLine($"bytesize:{capacityInBytes}time:{frame.Duration.GetValueOrDefault().TotalMilliseconds}"); + } + + + } + + + public async void WriteIntoMemory(IBuffer buffer) + { + + var x = await _memoryStream.WriteAsync(buffer); + } + } } diff --git a/SimpleRecorder/Package.appxmanifest b/SimpleRecorder/Package.appxmanifest index ecfec7b..7d71832 100644 --- a/SimpleRecorder/Package.appxmanifest +++ b/SimpleRecorder/Package.appxmanifest @@ -24,5 +24,7 @@ + + \ No newline at end of file diff --git a/SimpleRecorder/SimpleRecorder.csproj b/SimpleRecorder/SimpleRecorder.csproj index 60fa6c0..157a990 100644 --- a/SimpleRecorder/SimpleRecorder.csproj +++ b/SimpleRecorder/SimpleRecorder.csproj @@ -11,7 +11,7 @@ SimpleRecorder en-US UAP - 10.0.19041.0 + 10.0.17763.0 10.0.17763.0 14 512 From d2422e2254452f666e230d69f89c052714ec1cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=8B=E4=BB=95=E9=92=8A?= Date: Fri, 25 Jun 2021 19:04:33 +0800 Subject: [PATCH 4/4] fix --- CaptureEncoder/AppSettingHelper.cs | 76 ----------- CaptureEncoder/CaptureEncoder.csproj | 1 - CaptureEncoder/Encoder.cs | 192 +++++---------------------- 3 files changed, 35 insertions(+), 234 deletions(-) delete mode 100644 CaptureEncoder/AppSettingHelper.cs diff --git a/CaptureEncoder/AppSettingHelper.cs b/CaptureEncoder/AppSettingHelper.cs deleted file mode 100644 index b5a8204..0000000 --- a/CaptureEncoder/AppSettingHelper.cs +++ /dev/null @@ -1,76 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Windows.Storage; - -namespace CaptureEncoder -{ - //public static class AppSettingHelper - //{ - // private static ApplicationDataContainer root = ApplicationData.Current.LocalSettings; - - // public static string key_RecordMike = "IsMikeRecord"; - // public static string key_InputDeviceIndex = "MikeRecordedIndex"; - // public static string key_OutputDeviceRecordedIndex = "OutputDeviceRecordedIndex"; - - // public static bool UseMikeAsInput - // { - // get - // { - // return ReadSetting(key_RecordMike); - - // } - // } - - // public static int InputDeviceIndex - // { - // get - // { - // return ReadSetting(key_InputDeviceIndex); - - // } - // } - - // public static int OutputDeviceRecordedIndex - // { - // get - // { - // return ReadSetting(key_OutputDeviceRecordedIndex); - - // } - // } - - - - // // - // public static T ReadSetting(string key) - // { - // if (root.Values.TryGetValue(key, out object value)) - // { - // return (T)value; - // } - // else - // { - // return default(T); - // } - // } - - // // - // public static void WriteSetting(string key, T Tvalue)// where T : struct - // { - // ApplicationDataContainer root = ApplicationData.Current.LocalSettings; - // if (root.Values.TryGetValue(key, out object oldkey)) - // { - - // root.Values[key] = Tvalue; - // } - // else - // { - // root.Values.Add(key, Tvalue); - // } - // } - - //} -} diff --git a/CaptureEncoder/CaptureEncoder.csproj b/CaptureEncoder/CaptureEncoder.csproj index c42b0c2..135a9e4 100644 --- a/CaptureEncoder/CaptureEncoder.csproj +++ b/CaptureEncoder/CaptureEncoder.csproj @@ -123,7 +123,6 @@ PackageReference - diff --git a/CaptureEncoder/Encoder.cs b/CaptureEncoder/Encoder.cs index aacdc8e..18a9120 100644 --- a/CaptureEncoder/Encoder.cs +++ b/CaptureEncoder/Encoder.cs @@ -37,11 +37,9 @@ public Encoder(IDirect3DDevice device, GraphicsCaptureItem item) private async Task CreateAudioObjects() { - // create AudioGraph AudioGraphSettings settings = new AudioGraphSettings(Windows.Media.Render.AudioRenderCategory.Media); settings.QuantumSizeSelectionMode = QuantumSizeSelectionMode.LowestLatency; - var outputDevices = await DeviceInformation.FindAllAsync(MediaDevice.GetAudioRenderSelector()); - settings.PrimaryRenderDevice = outputDevices[0]; + // create AudioGraph var result = await AudioGraph.CreateAsync(settings); if (result.Status != AudioGraphCreationStatus.Success) { @@ -49,58 +47,22 @@ private async Task CreateAudioObjects() return; } _audioGraph = result.Graph; - - _audioGraph.UnrecoverableErrorOccurred += (sender,e) => { - sender.Dispose(); - }; - //_audioGraph.QuantumStarted += _audioGraph_QuantumStarted; - // create device output - var deviceOutputResult = await _audioGraph.CreateDeviceOutputNodeAsync(); - if (deviceOutputResult.Status != AudioDeviceNodeCreationStatus.Success) - { - Debug.WriteLine("Cannot create device output node"); - return; - } - _deviceOutputNode= deviceOutputResult.DeviceOutputNode; - - // create frame output - _frameOutputNode = _audioGraph.CreateFrameOutputNode(); - //_frameOutputNode.ConsumeInput = true; - - //_frameOutputNode.Start(); - // create device input + // create device input _ a microphone var deviceInputResult = await _audioGraph.CreateDeviceInputNodeAsync(MediaCategory.Other); if (deviceInputResult.Status != AudioDeviceNodeCreationStatus.Success) { Debug.WriteLine($"Audio Device Input unavailable because {deviceInputResult.Status.ToString()}"); - + return; } _deviceInputNode = deviceInputResult.DeviceInputNode; - _deviceInputNode.AddOutgoingConnection(_deviceOutputNode); - //_deviceInputNode.AddOutgoingConnection(_frameOutputNode); - - //mp3File = await KnownFolders.VideosLibrary.CreateFileAsync("temp.mp3", CreationCollisionOption.ReplaceExisting); - //var fileOutputNodeResult=await _audioGraph.CreateFileOutputNodeAsync(mp3File, MediaEncodingProfile.CreateMp3(AudioEncodingQuality.High)); - //if (fileOutputNodeResult.Status != AudioFileNodeCreationStatus.Success) - //{ - // Debug.WriteLine($"Audio File output unavailable because {deviceInputResult.Status.ToString()}"); - - // return; - //} - - //_fileOutputNode = fileOutputNodeResult.FileOutputNode; - - - // - - - - - - //_deviceInputNode.AddOutgoingConnection(_fileOutputNode); + // create output frame + _frameOutputNode = _audioGraph.CreateFrameOutputNode(); + // increase volume of input + _deviceInputNode.OutgoingGain = 10; + _deviceInputNode.AddOutgoingConnection(_frameOutputNode); } @@ -133,19 +95,22 @@ private async Task EncodeInternalAsync(IRandomAccessStream stream, uint width, u encodingProfile.Video.FrameRate.Denominator = 1; encodingProfile.Video.PixelAspectRatio.Numerator = 1; encodingProfile.Video.PixelAspectRatio.Denominator = 1; + // Describe audio input + encodingProfile.Audio = MediaEncodingProfile.CreateMp3(AudioEncodingQuality.Low).Audio; - // Describe audio input - await CreateAudioObjects(); + // create audio graph + if (_audioGraph==null) + { + await CreateAudioObjects(); + } + + // add audio support _audioDescriptor = new AudioStreamDescriptor(_audioGraph.EncodingProperties); _mediaStreamSource.AddStreamDescriptor(_audioDescriptor); - encodingProfile.Audio = MediaEncodingProfile.CreateFlac(AudioEncodingQuality.Low).Audio; - - var transcode = await _transcoder.PrepareMediaStreamSourceTranscodeAsync(_mediaStreamSource, stream, encodingProfile); - await transcode.TranscodeAsync(); } } @@ -176,8 +141,6 @@ private void DisposeInternal() private void CreateMediaObjects() { - - // Create our encoding profile based on the size of the item int width = _captureItem.Size.Width; int height = _captureItem.Size.Height; @@ -186,12 +149,10 @@ private void CreateMediaObjects() var videoProperties = VideoEncodingProperties.CreateUncompressed(MediaEncodingSubtypes.Bgra8, (uint)width, (uint)height); _videoDescriptor = new VideoStreamDescriptor(videoProperties); - // audio - - // Create our MediaStreamSource _mediaStreamSource = new MediaStreamSource(_videoDescriptor); - _mediaStreamSource.BufferTime = TimeSpan.FromSeconds(0); + _mediaStreamSource.CanSeek = true; + _mediaStreamSource.BufferTime = TimeSpan.FromMilliseconds(0); _mediaStreamSource.Starting += OnMediaStreamSourceStarting; _mediaStreamSource.SampleRequested += OnMediaStreamSourceSampleRequested; _mediaStreamSource.Closed += (s,e) => { @@ -200,14 +161,12 @@ private void CreateMediaObjects() }; - - // Create our transcoder _transcoder = new MediaTranscoder(); _transcoder.HardwareAccelerationEnabled = true; } - TimeSpan delay=new TimeSpan(0); + unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, MediaStreamSourceSampleRequestedEventArgs args) { if (_isRecording && !_closed) @@ -218,7 +177,6 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, if (args.Request.StreamDescriptor.GetType() == typeof(VideoStreamDescriptor)) { // Request Video - using (var frame = _frameGenerator.WaitForNewFrame()) { if (frame == null) @@ -227,10 +185,7 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, DisposeInternal(); return; } - - var timeStamp = frame.SystemRelativeTime; - //Debug.WriteLine($"video:{timeStamp.TotalMilliseconds}"); - //_time = timeStamp; + var timeStamp = frame.SystemRelativeTime- timeOffset; var sample = MediaStreamSample.CreateFromDirect3D11Surface(frame.Surface, timeStamp); args.Request.Sample = sample; } @@ -241,9 +196,12 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, var deferal = request.GetDeferral(); - var frame = GetFrameAsync(); - - + var frame = _frameOutputNode.GetFrame(); + if (frame.Duration.GetValueOrDefault().TotalSeconds==0) + { + args.Request.Sample = null; + return; + } using (AudioBuffer buffer = frame.LockBuffer(AudioBufferAccessMode.Write)) using (IMemoryBufferReference reference = buffer.CreateReference()) { @@ -255,28 +213,19 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, Marshal.Copy((IntPtr)dataInBytes, bytes, 0, (int)capacityInBytes); var data_buffer = WindowsRuntimeBufferExtensions.AsBuffer(bytes, 0, (int)capacityInBytes); - var stamp = time_start + frame.RelativeTime.GetValueOrDefault(); + var stamp = frame.RelativeTime.GetValueOrDefault(); var duration = frame.Duration.GetValueOrDefault(); var sample = MediaStreamSample.CreateFromBuffer(data_buffer, stamp); - sample.Duration = duration;// frame.Duration.GetValueOrDefault(); + sample.Duration = duration; sample.KeyFrame = true; - - if (sample.Discontinuous) - { - Debug.WriteLine("lost sample"); - sample.Discontinuous = false; - } - //Debug.WriteLine($"audio:{stamp.TotalMilliseconds}duration:{sample.Duration.TotalMilliseconds}"); request.Sample = sample; - //Debug.WriteLine($"bytesize:{capacityInBytes}time:{frame.Duration.GetValueOrDefault().TotalMilliseconds}"); + } deferal.Complete(); - - } } @@ -297,58 +246,21 @@ unsafe private void OnMediaStreamSourceSampleRequested(MediaStreamSource sender, } - - AudioFrame GetFrameAsync() { - var frame = _frameOutputNode.GetFrame(); - if (frame.Duration.GetValueOrDefault().TotalSeconds != 0) - { - return frame; - - } - else - { - Debug.Write("Delay"); - Task.Delay(5); - return GetFrameAsync(); - } - } - - - - - TimeSpan time_start=new TimeSpan(); + private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSourceStartingEventArgs args) { MediaStreamSourceStartingRequest request = args.Request; - using (var frame = _frameGenerator.WaitForNewFrame()) { - time_start = frame.SystemRelativeTime; - request.SetActualStartPosition(frame.SystemRelativeTime); - + timeOffset = frame.SystemRelativeTime; + //request.SetActualStartPosition(frame.SystemRelativeTime); } _audioGraph?.Start(); using (var audioFrame = _frameOutputNode.GetFrame()) { - time_start= time_start-audioFrame.RelativeTime.GetValueOrDefault(); + timeOffset = timeOffset + audioFrame.RelativeTime.GetValueOrDefault(); } - //if ((request.StartPosition != null)) - //{ - // UInt64 sampleOffset = (UInt64)request.StartPosition.Value.Ticks / (UInt64)sampleDuration.Ticks; - // timeOffset = new TimeSpan((long)sampleOffset * sampleDuration.Ticks); - // byteOffset = sampleOffset * sampleSize; - // Debug.WriteLine($"timeOffset:{timeOffset.TotalMilliseconds}ms"); - //} - - - - - - - - - } private IDirect3DDevice _device; @@ -367,41 +279,7 @@ private void OnMediaStreamSourceStarting(MediaStreamSource sender, MediaStreamSo private AudioGraph _audioGraph; private AudioDeviceInputNode _deviceInputNode; private AudioFrameOutputNode _frameOutputNode; - private AudioDeviceOutputNode _deviceOutputNode; - private AudioFileOutputNode _fileOutputNode; - private const UInt32 sampleSize = 960; - private TimeSpan sampleDuration = TimeSpan.FromMilliseconds(10); - private InMemoryRandomAccessStream _memoryStream = new InMemoryRandomAccessStream(); - private IRandomAccessStream _fileStream; - - unsafe private void _audioGraph_QuantumStarted(AudioGraph sender, object args) - { - AudioFrame frame = _frameOutputNode?.GetFrame(); - - using (AudioBuffer buffer = frame.LockBuffer(AudioBufferAccessMode.Write)) - using (IMemoryBufferReference reference = buffer.CreateReference()) - { - byte* dataInBytes; - uint capacityInBytes; - // Get the buffer from the AudioFrame - ((IMemoryBufferByteAccess)reference).GetBuffer(out dataInBytes, out capacityInBytes); - byte[] bytes = new byte[capacityInBytes]; - Marshal.Copy((IntPtr)dataInBytes, bytes, 0, (int)capacityInBytes); - var data_buffer = WindowsRuntimeBufferExtensions.AsBuffer(bytes, 0, (int)capacityInBytes); - - WriteIntoMemory(data_buffer); - //Debug.WriteLine($"bytesize:{capacityInBytes}time:{frame.Duration.GetValueOrDefault().TotalMilliseconds}"); - } - - - } - - - public async void WriteIntoMemory(IBuffer buffer) - { - - var x = await _memoryStream.WriteAsync(buffer); - } + private TimeSpan timeOffset = new TimeSpan(); } }