diff --git a/doc/UMAP.rst b/doc/UMAP.rst index 50a5f186..d310fb9a 100644 --- a/doc/UMAP.rst +++ b/doc/UMAP.rst @@ -21,7 +21,7 @@ :control numNeighbours: - The number of neighbours considered by the algorithm to balance local vs global structures to conserve. Low values will prioritise preservation of the local structure while high values will prioritise preservation of the global structure. + The number of neighbours considered by the algorithm to balance local vs global structures to conserve. Low values will prioritise preserving local structure, high values will help preserve the global structure. :control minDist: diff --git a/example-code/sc/AudioTransport.scd b/example-code/sc/AudioTransport.scd index 4c1ec110..960f5921 100644 --- a/example-code/sc/AudioTransport.scd +++ b/example-code/sc/AudioTransport.scd @@ -1,27 +1,64 @@ code:: -//didactic - the mouse X axis interpolates between the two sinewaves -{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr())}.play; -//notice how the interpolation quantizes to the FFT bins. Like most spectral processes, it benefits from oversampling the fft... at the cost of CPU power, obviously. -{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr(),fftSize: 8192)}.play; +( +~srcA = Buffer.readChannel(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav"),channels:[0]); +~srcB = Buffer.readChannel(s,FluidFilesPath("Nicol-LoopE-M.wav"),channels:[0]); +) -// when the signal is steady, larger hopSize can be accommodated to save back on the CPU -{FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1),SinOsc.ar(440,mul: 0.02),MouseX.kr(),windowSize: 8192)}.play; // here we get a default hop of half the window so 8 times less than above. +( +{ + var srcA = PlayBuf.ar(1,~srcA,BufRateScale.ir(~srcA),loop:1); + var srcB = PlayBuf.ar(1,~srcB,BufRateScale.ir(~srcB),loop:1); + FluidAudioTransport.ar(srcA,srcB,MouseX.kr,1024,64,2048).dup; +}.play +) -//if you CPU can cope, try this setting, almost smooth, but attacks would smear (the Y axis mixes some in to hear the effect) -{var attacks = Impulse.ar(1,mul: MouseY.kr(-40,10).dbamp); FluidAudioTransport.ar(SinOsc.ar(220,mul: 0.1,add: attacks),SinOsc.ar(440,mul: 0.02,add: attacks),MouseX.kr(),windowSize: 16000)}.play; +:: +strong::The impact of FFT Settings:: +The larger the FFT size the better it blends the spectral qualities at the expense of smearing attacks. This will have a drastic impact on the characteristic of the interpolation. +code:: + +( +~srcA = Buffer.readChannel(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav"),channels:[0]); +~srcB = Buffer.readChannel(s,FluidFilesPath("Nicol-LoopE-M.wav"),channels:[0]); +) -//richer with complex spectra -//load 2 files ( -b = Buffer.read(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-CF-ChurchBells.wav")); +~synth = { + arg windowSize = 1024, hopSize = 64, fftSize = 2048; + var srcA = PlayBuf.ar(1,~srcA,BufRateScale.ir(~srcA),loop:1); + var srcB = PlayBuf.ar(1,~srcB,BufRateScale.ir(~srcB),loop:1); + FluidAudioTransport.ar(srcA,srcB,MouseX.kr,windowSize,hopSize,fftSize,16384).dup; +}.play ) -//listen to them -b.play -c.play -//stereo cross! -{FluidAudioTransport.ar(PlayBuf.ar(2,b,loop: 1),PlayBuf.ar(2,c,loop: 1),MouseX.kr())}.play; + +// Try these different FFT settings to see how they affect the results of the audio transport algorithm + +~synth.set(\windowSize,256,\hopSize,16,\fftSize,256); +~synth.set(\windowSize,1024,\hopSize,16,\fftSize,1024); +~synth.set(\windowSize,1024,\hopSize,-1,\fftSize,-1); +~synth.set(\windowSize,2048,\hopSize,-1,\fftSize,-1); +~synth.set(\windowSize,4096,\hopSize,-1,\fftSize,-1); +~synth.set(\windowSize,16384,\hopSize,-1,\fftSize,-1); :: +Creative Modulation +code:: + +( +~srcA = Buffer.readChannel(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav"),channels:[0]); +~srcB = Buffer.readChannel(s,FluidFilesPath("Nicol-LoopE-M.wav"),channels:[0]); +) + +( +{ + var srcA = PlayBuf.ar(1,~srcA,BufRateScale.ir(~srcA),loop:1); + var srcB = PlayBuf.ar(1,~srcB,BufRateScale.ir(~srcB),loop:1); + var amp = Amplitude.kr(srcB); + var interp = LFDNoise3.kr(100 * amp).range(0,1).poll; + var sig = FluidAudioTransport.ar(srcA,srcB,interp,2048).dup; + sig + (srcB * -20.dbamp); +}.play +) +:: \ No newline at end of file diff --git a/example-code/sc/BufAudioTransport.scd b/example-code/sc/BufAudioTransport.scd index b5fab464..9cce89ec 100644 --- a/example-code/sc/BufAudioTransport.scd +++ b/example-code/sc/BufAudioTransport.scd @@ -1,36 +1,27 @@ code:: -//Didactic: -//Make 2 sinewave sources to be interpolated ( -b = Buffer.loadCollection(s, FloatArray.fill(44100, {|a|(a / pi).sin * 0.1})); -c = Buffer.loadCollection(s, FloatArray.fill(44100, {|a|(a / pi / 2).sin * 0.02})); -d = Buffer.new +~srcA = Buffer.readChannel(s,FluidFilesPath("Tremblay-CEL-GlitchyMusicBoxMelo.wav"),channels:[0]); +~srcB = Buffer.readChannel(s,FluidFilesPath("Nicol-LoopE-M.wav"),channels:[0]); ) -//make an sound interpolating their spectrum -FluidBufAudioTransport.process(s,b,source2:c,destination:d,interpolation:0.5,action:{"Ding".postln}) - -// listen to the source and the result -b.play -c.play -d.play - -// note that the process is quantized by the spectral bins. For an example of the pros and cons of these settings on this given process, please see the realtime FluidAudioTransport helpfile. - -// more interesting sources: two cardboard bowing gestures +// fft settings have a large impact on the results. play around with them to hear how. +// and/or see the (real time) FluidAudioTransport example code ( -b = Buffer.read(s,FluidFilesPath("Green-Box641.wav")); -c = Buffer.read(s,FluidFilesPath("Green-Box639.wav")); -d = Buffer.new +~result = Buffer(s); +FluidBufAudioTransport.processBlocking(s, + sourceA:~srcA, + sourceB:~srcB, + destination:~result, + interpolation:0.5, + windowSize:1024, // this is the default + hopSize:512, // this is the default + fftSize:1024, // this is the default + action:{"done".postln;} +); ) -// listen to the source -b.play -c.play +// the output will be the same length as the shorter source buffer +~result.play; -// process and listen -FluidBufAudioTransport.process(s,b,source2:c,destination:d,interpolation:0.5,action:{"Ding".postln}) -d.play -// try various interpolation factors (0.1 and 0.9 are quite good :: diff --git a/example-code/sc/BufChroma.scd b/example-code/sc/BufChroma.scd index d7b1c80b..0d9199c1 100644 --- a/example-code/sc/BufChroma.scd +++ b/example-code/sc/BufChroma.scd @@ -1,50 +1,82 @@ code:: -// create some buffers -( -b = Buffer.read(s,FluidFilesPath("Tremblay-SlideChoirAdd-M.wav")); -c = Buffer.new(s); -) -// run the process with basic parameters +~src = Buffer.read(s,FluidFilesPath("Harker-DS-TenOboeMultiphonics-M.wav")); + +// listen if you want +~src.play; + ( -Routine{ - t = Main.elapsedTime; - FluidBufChroma.process(s, b, features: c, windowSize: 4096).wait; - (Main.elapsedTime - t).postln; -}.play +fork({ + var win, synth; + var chroma = Buffer(s); + + FluidBufChroma.processBlocking(s,~src,features:chroma); + + s.sync; + + synth = { + arg index = 0; + var chromaVector = BufRd.kr(12,chroma,index,1,1); + var freqs = 440 * 12.collect{arg i; 2.pow(i/12)}; + var sig = SinOsc.ar(freqs,0,chromaVector); + Splay.ar(sig); + }.play; + + win = Window("FluidChroma: click me to hear sines of the chroma analysis",Rect(0,0,1600,400)); + + FluidWaveform(~src,featuresBuffer:chroma,parent:win,bounds:win.bounds,stackFeatures:true,standalone:false); + + UserView(win,win.bounds).mouseDownAction_{ + arg view, x, y; + var index = x.linlin(0,view.bounds.width,0,chroma.numFrames-1).asInteger; + synth.set(\index,index); + }; + + win.front; +},AppClock) ) -// listen to the source and look at the buffer -b.play; -c.plot :: +strong::Dividing the octave into more than 12 divisions:: +code:: + +// doing a chroma analysis dividing the octvae into 19 equal divisions: -STRONG::A stereo buffer example.:: -CODE:: +~src = Buffer.read(s,FluidFilesPath("Harker-DS-TenOboeMultiphonics-M.wav")); + +// listen if you want +~src.play; -// load two very different files ( -b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); -) +fork({ + var win, synth; + var chroma = Buffer(s); -// composite one on left one on right as test signals -FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b) -b.play + FluidBufChroma.processBlocking(s,~src,features:chroma,numChroma:19); -// create a buffer as destinations -c = Buffer.new(s); + s.sync; -//run the process on them -( -Routine{ - t = Main.elapsedTime; - FluidBufChroma.process(s, b, features: c, windowSize: 4096).wait; - (Main.elapsedTime - t).postln; -}.play + synth = { + arg index = 0; + var chromaVector = BufRd.kr(chroma.numChannels,chroma,index,1,1); + var freqs = 440 * chroma.numChannels.collect{arg i; 2.pow(i/chroma.numChannels)}; + var sig = SinOsc.ar(freqs,0,chromaVector); + Splay.ar(sig); + }.play; + + win = Window("FluidChroma: click me to hear sines of the chroma analysis",Rect(0,0,1600,400)); + + FluidWaveform(~src,featuresBuffer:chroma,parent:win,bounds:win.bounds,stackFeatures:true,standalone:false); + + UserView(win,win.bounds).mouseDownAction_{ + arg view, x, y; + var index = x.linlin(0,view.bounds.width,0,chroma.numFrames-1).asInteger; + synth.set(\index,index); + }; + + win.front; +},AppClock) ) -// look at the buffer: 12 chroma bins for left, then 12 chroma bins for right -c.plot(separately:true) :: diff --git a/example-code/sc/BufCompose.scd b/example-code/sc/BufCompose.scd index 9def3cee..2838cec1 100644 --- a/example-code/sc/BufCompose.scd +++ b/example-code/sc/BufCompose.scd @@ -1,50 +1,97 @@ +strong::Basic Copy:: +code:: +~src = Buffer.read(s,FluidFilesPath("Tremblay-FMTriDist-M.wav")); + +( +~destination = Buffer(s); +FluidBufCompose.processBlocking(s,~src,destination:~destination,action:{ + defer{ + ~src.plot; + ~destination.plot; + }; +}); +) +:: +strong::Mixing:: +code:: +( +~srcA = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcBassGuit-Melo-M.wav")); +~srcB = Buffer.read(s,FluidFilesPath("Tremblay-AaS-VoiceQC-B2K-M.wav")); +) +// listen separately if you want +~srcA.play; +~srcB.play; + +( +fork{ + ~destination = Buffer(s); + [~srcA,~srcB].do{ + arg src; + FluidBufCompose.processBlocking(s,src,destination:~destination,destGain:1,gain:-6.dbamp); + }; + s.sync; + ~destination.play; +} +) +:: +strong::Subsections:: code:: -// load some buffers ( -b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); -d = Buffer.new(s); +~srcA = Buffer.read(s,FluidFilesPath("Tremblay-SlideChoirAdd-M.wav")); +~srcB = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); ) -// with basic params (basic summing of each full buffer in all dimensions) +// listen separately if you want +~srcA.play; +~srcB.play; + ( -Routine{ - FluidBufCompose.process(s, source: b, destination: d); - FluidBufCompose.process(s, source: c, destination: d, destGain: 1.0); +fork{ + ~destination = Buffer(s); + + // starting at frame 30000, copy 1 second of ~srcB + FluidBufCompose.processBlocking(s,~srcB,startFrame:30000,numFrames:44100,destination:~destination,destGain:1,gain:-6.dbamp); + // copy 9000 frames of ~srcA + FluidBufCompose.processBlocking(s,~srcA,numFrames:9000,destination:~destination,destGain:1,gain:-6.dbamp); + s.sync; - d.query; - d.play; -}.play; -) -//constructing a mono buffer, with a quiet punch from the synth, with a choked piano resonance from the left channel -( -Routine{ - d.free; d = Buffer.new(s); - FluidBufCompose.process(s, source: b, numFrames: 9000, gain: 0.5, destination: d); - FluidBufCompose.process(s, source: c, startFrame:30000, numFrames:44100, numChans:1, gain:0.9, destination: d, destGain: 1.0).wait; - d.query; - d.play; -}.play -) -//constructing a stereo buffer, with the end of the mono synth in both channels, with a piano resonance in swapped stereo -( -Routine{ - d.free; d = Buffer.new(s); - FluidBufCompose.process(s, source: b, startFrame: 441000, numChans: 2, gain: 0.6, destination: d); - FluidBufCompose.process(s, source: c, numFrames: 78000, startChan: 1, numChans: 2, gain: 0.5, destStartFrame: 22050, destination: d, destGain: 1.0).wait; - d.query; - d.play; -}.play -) -//constructing a one second buffer: the first second of each buffer, the mono synth on the right, the piano on the left -( -Routine{ - d.free; d = Buffer.new(s); - FluidBufCompose.process(s, source: b, numFrames: 44100, numChans: 1, destStartChan: 1, destination: d); - FluidBufCompose.process(s, source: c, numFrames:44100, numChans:1, destination: d, destGain: 1.0).wait; - d.query; - d.play; -}.play + ~destination.play; +} ) :: +strong::Multichannel:: +code:: + +( +~stereoPiano = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); +~synthSounds = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); +) + +// copy the mono synth sounds into the R channel of the stereo piano buffer using destStartChan:1 +( +FluidBufCompose.processBlocking(s,~synthSounds,destination:~stereoPiano,destGain:1,destStartChan:1,action:{ + ~stereoPiano.play; +}); +) + +// copy just the L channel of the piano into the R channel of the synth sounds +// (because it doesn't have an R channel yet, BufCompose will create one) +( +FluidBufCompose.processBlocking(s,~stereoPiano,destination:~synthSounds,numChans:1,destGain:1,destStartChan:1,action:{ + ~synthSounds.play; +}); +) + +// if numChans is greater than the available channels it will wrap around to the beginning channels, so we can reverse the channels like this: + +// original +~stereoPiano.play; + +( +~reversed = Buffer(s); +FluidBufCompose.processBlocking(s,~stereoPiano,startChan:1,numChans:2,destination:~reversed,action:{ + ~reversed.play; +}); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufLoudness.scd b/example-code/sc/BufLoudness.scd index 2ce15113..2fca4534 100644 --- a/example-code/sc/BufLoudness.scd +++ b/example-code/sc/BufLoudness.scd @@ -1,59 +1,42 @@ code:: -// create a buffer with a short clicking sinusoidal burst (220Hz) starting at frame 8192 for 1024 frames -( -b = Buffer.sendCollection(s, (Array.fill(8192,{0}) ++ (Signal.sineFill(1203,[0,0,0,0,0,1],[0,0,0,0,0,0.5pi]).takeThese({|x,i|i>1023})) ++ Array.fill(8192,{0}))); -c = Buffer.new(s); -) -// listen to the source and look at the buffer -b.play; b.plot; +~src = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); -// run the process with basic parameters +// blue is loudness dBFS, orange is true peak dBFS ( -Routine{ - t = Main.elapsedTime; - FluidBufLoudness.process(s, source:b, features: c).wait; - (Main.elapsedTime - t).postln; -}.play +~loudness = Buffer(s); +FluidBufLoudness.processBlocking(s,~src,features:~loudness); +FluidWaveform(~src,featuresBuffer:~loudness,bounds:Rect(0,0,1600,400),stackFeatures:true); +~src.play; ) -// look at the analysis -c.plot(minval:-130, maxval:6) - -// The values are interleaved [loudness,truepeak] in the buffer as they are on 2 channels: to get to the right frame, divide the SR of the input by the hopSize, then multiply by 2 because of the channel interleaving -// here we are querying from one frame before (the signal starts at 8192, which is frame 16 (8192/512), therefore starting the query at frame 15, which is index 30. - -c.getn(30,10,{|x|x.postln}) - -// observe that the first frame is silent, as expected. We can appreciate the overshoot of TruePeak of a full range sinewave starting on the second sample (fourth item in the list). :: +strong::Getting some useful loudness stats on an audio file (that is in a Buffer):: +As a result of the loudness wars, a statistical understanding of the loudness of an audio file (such as a rendered track) may be useful. Here are some ways you might use FluidBufLoudness this way. -STRONG::A stereo buffer example.:: -CODE:: - -// load two very different files -( -b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); -) - -// composite one on left one on right as test signals -FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b) -b.play +The EBU standard specifies that the window should be 400ms long, and update every 100ms, for instantaneous loudness. Here we set the windowSize and hopSize appropriately using the sample rate of the source buffer. +code:: -// create a buffer as destinations -c = Buffer.new(s); +// stereo song +~src = Buffer.read(s,FluidFilesPath("Tremblay-beatRemember.wav")); -//run the process on them with EBU standard Instant Loudness of ( -Routine{ - t = Main.elapsedTime; - FluidBufLoudness.process(s, b, features: c, windowSize: 17640, hopSize:4410).wait; - (Main.elapsedTime - t).postln; -}.play +~loudness = Buffer(s); +FluidBufLoudness.processBlocking(s,~src,features:~loudness,windowSize:~src.sampleRate*0.4,hopSize:~src.sampleRate*0.1); +~stats = Buffer(s); +FluidBufStats.processBlocking(s,~loudness,stats:~stats); +~stats.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~stats.numChannels).flop; + "L channel mean loudness: % dB".format(fa[0][0]).postln; + "L channel median loudness: % dB".format(fa[0][5]).postln; + "L channel max true peak: % dB".format(fa[1][6]).postln; + + "R channel mean loudness: % dB".format(fa[2][0]).postln; + "R channel median loudness: % dB".format(fa[2][5]).postln; + "R channel max true peak: % dB".format(fa[3][6]).postln; +}); ) -// look at the buffer: [loudness,truepeak] for left then [loudness,truepeak] for right -c.plot(minval:-40, maxval:0) -:: +:: \ No newline at end of file diff --git a/example-code/sc/BufNoveltySlice.scd b/example-code/sc/BufNoveltySlice.scd index f1e21544..e0b2fa1d 100644 --- a/example-code/sc/BufNoveltySlice.scd +++ b/example-code/sc/BufNoveltySlice.scd @@ -1,99 +1,28 @@ - code:: -// load some buffers -( -b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); -c = Buffer.new(s); -) - -( -// with basic params, with a minimum slight length to avoid over -Routine{ - t = Main.elapsedTime; - FluidBufNoveltySlice.process(s,b, indices: c, threshold:0.4,filterSize: 4, minSliceLength: 8).wait; - (Main.elapsedTime - t).postln; -}.play -) -//check the number of slices: it is the number of frames in the transBuf minus the boundary index. -c.query; +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -//loops over a splice with the MouseX ( -{ - BufRd.ar(1, b, - Phasor.ar(0,1, - BufRd.kr(1, c, - MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1), - BufRd.kr(1, c, - MouseX.kr(1, BufFrames.kr(c)), 0, 1), - BufRd.kr(1,c, - MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1); - }.play; -) - :: - -STRONG::Examples of the impact of the filterSize:: - - CODE:: -// load some buffers -( -b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); -c = Buffer.new(s); +~indices = Buffer(s); +FluidBufNoveltySlice.processBlocking(s,~src,indices:~indices); +FluidWaveform(~src,~indices,bounds:Rect(0,0,1600,400)); ) -// process with a given filterSize -( -Routine{ - FluidBufNoveltySlice.process(s,b, indices: c, kernelSize:31, threshold:0.1, filterSize:1).wait; -//check the number of slices: it is the number of frames in the transBuf minus the boundary index. - c.query; -}.play; -) -//play slice number 3 +//loops over a slice from onset to the next slice point using MouseX to choose which ( { - BufRd.ar(1, b, - Line.ar( - BufRd.kr(1, c, DC.kr(3), 0, 1), - BufRd.kr(1, c, DC.kr(4), 0, 1), - (BufRd.kr(1, c, DC.kr(4)) - BufRd.kr(1, c, DC.kr(3), 0, 1) + 1) / s.sampleRate), - 0,1); + var gate_index = MouseX.kr(0,~indices.numFrames-1).poll(label:"slice index"); + var start = BufRd.kr(1,~indices,gate_index,1,1); + var end = BufRd.kr(1,~indices,gate_index+1,1,1); + var phs = Phasor.ar(0,BufRateScale.ir(~src),start,end); + BufRd.ar(1,~src,phs,1,4).dup; }.play; ) -// change the filterSize in the code above to 4. Then to 12. Listen in between to the differences. - -// What's happening? In the first instance (filterSize = 1), the novelty line is jittery and therefore overtriggers on the arpegiated guitar. We also can hear attacks at the end of the segment. Setting the threshold higher (like in the 'Basic Example' pane) misses some more subtle variations. - -// So in the second settings (filterSize = 4), we smooth the novelty line a little, which allows us to catch small differences that are not jittery. It also corrects the ending cutting by the same trick: the averaging of the sharp pick is sliding up, crossing the threshold slightly earlier. - -// If we smooth too much, like the third settings (filterSize = 12), we start to loose precision and miss attacks. Have fun with different values of theshold then will allow you to find the perfect segment for your signal. -:: - -STRONG::A stereo buffer example.:: -CODE:: - -// make a stereo buffer -b = Buffer.alloc(s,88200,2); - -// add some stereo clicks and listen to them -((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)}); -b.play - -// create a new buffer as destinations -c = Buffer.new(s); - -//run the process on them +// instead of the raw spectrum (algorithm:0), we'll try mfccs (algorithm:1): ( -// with basic params -Routine{ - t = Main.elapsedTime; - FluidBufNoveltySlice.process(s,b, indices: c, threshold:0.3).wait; - (Main.elapsedTime - t).postln; -}.play +~indices = Buffer(s); +FluidBufNoveltySlice.processBlocking(s,~src,algorithm:1,threshold:0.5,kernelSize:5,filterSize:5,indices:~indices,minSliceLength:5); +FluidWaveform(~src,~indices,bounds:Rect(0,0,1600,400)); ) - -// list the indicies of detected attacks - the two input channels have been summed -c.getn(0,c.numFrames,{|item|(item * 2).postln;}) :: diff --git a/example-code/sc/BufOnsetSlice.scd b/example-code/sc/BufOnsetSlice.scd index ea0dc4ec..cb0474dd 100644 --- a/example-code/sc/BufOnsetSlice.scd +++ b/example-code/sc/BufOnsetSlice.scd @@ -1,61 +1,28 @@ - CODE:: -( -//prep some buffers -b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -c = Buffer.new(s); -) - -( -// with basic params -Routine{ - t = Main.elapsedTime; - FluidBufOnsetSlice.process(s,b, indices: c, threshold:0.5).wait; - (Main.elapsedTime - t).postln; -}.play -) -//check the number of slices: it is the number of frames in the transBuf minus the boundary index. -c.query; +~src = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); -//loops over a splice with the MouseX +// slice with basic parameters ( -{ - BufRd.ar(1, b, - Phasor.ar(0,1, - BufRd.kr(1, c, - MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1), - BufRd.kr(1, c, - MouseX.kr(1, BufFrames.kr(c)), 0, 1), - BufRd.kr(1,c, - MouseX.kr(0, BufFrames.kr(c) - 1), 0, 1)), 0, 1); - }.play; +~indices = Buffer(s); +FluidBufOnsetSlice.processBlocking(s,~src,metric:9,indices:~indices,threshold:0.3,action:{ + "found % slice points".format(~indices.numFrames).postln; + "with an average length of % seconds per slice".format(~src.duration / ~indices.numFrames).postln; +}); ) -:: - -STRONG::A stereo buffer example.:: -CODE:: -// make a stereo buffer -b = Buffer.alloc(s,88200,2); +// display +FluidWaveform(~src,~indices); -// add some stereo clicks and listen to them -((0..3)*22050+11025).do({|item,index| b.set(item+(index%2), 1.0)}) -b.play - -// create a new buffer as destinations -c = Buffer.new(s); - -//run the process on them +//loops over a slice from onset to the next slice point using MouseX to choose which ( -// with basic params -Routine{ - t = Main.elapsedTime; - FluidBufOnsetSlice.process(s,b, indices: c, threshold:0.00001).wait; - (Main.elapsedTime - t).postln; -}.play +{ + var gate_index = MouseX.kr(0,~indices.numFrames-1).poll(label:"slice index"); + var start = BufRd.kr(1,~indices,gate_index,1,1); + var end = BufRd.kr(1,~indices,gate_index+1,1,1); + var phs = Phasor.ar(0,BufRateScale.ir(~src),start,end); + BufRd.ar(1,~src,phs,1,4).dup; +}.play; ) -// list the indicies of detected attacks - the two input channels have been summed -c.getn(0,c.numFrames,{|item|(item * 2).postln;}) -:: +:: \ No newline at end of file diff --git a/example-code/sc/BufScale.scd b/example-code/sc/BufScale.scd index 41ec5fb5..946244bf 100644 --- a/example-code/sc/BufScale.scd +++ b/example-code/sc/BufScale.scd @@ -1,48 +1,39 @@ - code:: -( -Routine{ - // make a buffer of known qualities - b = Buffer.sendCollection(s,1.0.series(1.1,2.0)); - // and a destination buffer - c = Buffer(s); - // play with the scaling - FluidBufScale.process(s, b, destination: c, inputLow: 0, inputHigh: 1, outputLow: 20, outputHigh:10).wait; - // retrieve the buffer and enjoy the results. - c.getn(0,10,{|x|x.round(0.000001).postln;}) -}.play -) -// also works in multichannel - explore the following buffer +~raw = Buffer.loadCollection(s,(1..5)); -//process ( -Routine{ - b = Buffer.sendCollection(s,-10.0.series(-9,10.0).scramble,2); - c = Buffer(s); - s.sync; - defer{b.plot(bounds:Rect(400,400,400,400)).plotMode_(\points).bounds}; - FluidBufScale.process(s, b, destination: c, inputLow: -20, inputHigh: 20, outputLow: 0, outputHigh:1).wait; - //enjoy - same shape, different range - defer{c.plot(bounds:Rect(800,400,400,400)).plotMode_(\points)}; -}.play; -) +~scaled = Buffer(s); +FluidBufScale.processBlocking(s,~raw,destination:~scaled,inputLow:1,inputHigh:5,outputLow:0,outputHigh:1,action:{ + ~raw.getn(0,5,{ + arg vals; + "raw: %".format(vals).postln; + }); -//also works with a subset of the input, resizing the output -( -Routine{ - b = Buffer.sendCollection(s,0.0.series(0.1,3.0).reshape(3,10).flop.flat,3); - c = Buffer(s); - s.sync; - defer{b.plot(separately: true,bounds:Rect(400,400,400,400)).plotMode_(\points)}; - //process - FluidBufScale.process(s, b, startFrame: 3,numFrames: 4,startChan: 1,numChans: 1, destination: c, inputLow: 0, inputHigh: 3, outputLow: 0, outputHigh:1).wait; - //enjoy - c.query; - c.getn(0,4,{|x|x.postln;}); - defer{c.plot(separately: true,bounds:Rect(800,400,400,400)).plotMode_(\points)}; -}.play + ~scaled.getn(0,5,{ + arg vals; + "scaled: %".format(vals).postln; + }); +}); ) :: +strong::With Clipping:: +code:: + +~raw = Buffer.loadCollection(s,(1..5)); +( +~scaled = Buffer(s); +FluidBufScale.processBlocking(s,~raw,destination:~scaled,inputLow:1,inputHigh:4,outputLow:0,outputHigh:1,clipping:3,action:{ + ~raw.getn(0,5,{ + arg vals; + "raw: %".format(vals).postln; + }); + ~scaled.getn(0,5,{ + arg vals; + "scaled: %".format(vals).postln; + }); +}); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufSelect.scd b/example-code/sc/BufSelect.scd index d6330e17..fdbbf801 100644 --- a/example-code/sc/BufSelect.scd +++ b/example-code/sc/BufSelect.scd @@ -1,25 +1,82 @@ code:: -//send a known collection where the value of each frame in each channel is encoded -//chan -b = Buffer.sendCollection(s,30.collect{|x| x.mod(6) + (x.div(6) * 0.1)},6) -//check the ranges (thus showing a plotter error...) -b.plot(separately: true).plotMode_(\points) -//you can also check the collection itself if in doubt -b.getToFloatArray(action: {|x|x.round(0.1).postln;}); -//let's make a destination buffer -c = Buffer(s); +// a buffer with 5 frames and 3 channels, +// where, for the values in the buffer, the tens column +// is the channel, and the ones column is the frame +( +~src = Buffer.loadCollection(s,Array.fill(5,{ + arg frame; + Array.fill(3,{ + arg chan; + (chan * 10 ) + frame; + }); +}).flat,3); +) -//using default values, we copy everything: -FluidBufSelect.process(s,b,c,action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); +// print it out and look to check: +( +~src.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~src.numChannels).flop; + fa.do{ + arg chan; + chan.postln; + }; +}); +) -//more powerful copying, resizing the destination accordingly -FluidBufSelect.process(s,b,c, indices: [1,3], channels: [2,4], action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); +// now we'll copy out just frames 0, 2, and 3 out of channels 0 and 2 +( +~destination = Buffer(s); +FluidBufSelect.processBlocking(s,~src,~destination,[0,2,3],[0,2],action:{ + ~destination.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~destination.numChannels).flop; + fa.do{ + arg chan; + chan.postln; + }; + }); +}); +) -//observe the order can be anything, and -1 (default) passes everything in that dimension -FluidBufSelect.process(s,b,c, indices: [ -1 ] , channels: [3, 1, 4], action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); +// also, it can rearrange channels and indices for you! +( +~destination = Buffer(s); +FluidBufSelect.processBlocking(s,~src,~destination,[0,3,2],[2,0],action:{ + ~destination.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~destination.numChannels).flop; + fa.do{ + arg chan; + chan.postln; + }; + }); +}); +) :: +strong::Curating Descriptor Data:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-Iterative-M.wav")); +( +~descriptors = [0,1,5]; // channels for centroid, spread, and flatness +~stats = [5,6]; // indices for mid and low percentiles (the default will be median and maximum) +~specbuf = Buffer(s); +~statsbuf = Buffer(s); +~selected = Buffer(s); +FluidBufSpectralShape.processBlocking(s,~src,features:~specbuf); +FluidBufStats.processBlocking(s,~specbuf,stats:~statsbuf); +FluidBufSelect.processBlocking(s,~statsbuf,~selected,~stats,~descriptors); +~selected.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~selected.numChannels).flop; + "\t\t\tmedian\t\t\t\tmaximum".postln; + ["centroid","spread ","flatness"].do{ + arg name, i; + "% : %".format(name,fa[i]).postln; + }; +}); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufSelectEvery.scd b/example-code/sc/BufSelectEvery.scd index 094ac47b..3822ec44 100644 --- a/example-code/sc/BufSelectEvery.scd +++ b/example-code/sc/BufSelectEvery.scd @@ -1,25 +1,62 @@ -Didactic code:: -//send a known collection where the value of each frame in each channel is encoded -//chan -b = Buffer.sendCollection(s,30.collect{|x| x.mod(6) + (x.div(6) * 0.1)},6) -//check the ranges (thus showing a plotter error...) -b.plot(separately: true).plotMode_(\points) -//you can also check the collection itself if in doubt -b.getToFloatArray(action: {|x|x.round(0.1).postln;}); +// a buffer with 5 frames and 3 channels, +// where, for the values in the buffer, the tens column +// is the channel, and the ones column is the frame +( +~src = Buffer.loadCollection(s,Array.fill(5,{ + arg frame; + Array.fill(3,{ + arg chan; + (chan * 10 ) + frame; + }); +}).flat,3); +) -//let's make a destination buffer -c = Buffer(s); +// print it out and look to check: +( +~src.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~src.numChannels).flop; + fa.do{ + arg chan; + chan.postln; + }; +}); +) -//using default values, we copy everything: -FluidBufSelectEvery.process(s,b, destination: c, action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); +// copy out the odd frames and the even channels +( +~destination = Buffer(s); +FluidBufSelectEvery.processBlocking(s,~src,1,frameHop:2,chanHop:2,destination:~destination,action:{ + ~destination.loadToFloatArray(action:{ + arg fa; + fa = fa.clump(~destination.numChannels).flop; + fa.do{ + arg chan; + chan.postln; + }; + }); +}); +) +:: +strong::Curating Descriptors:: +code:: -//more powerful copying, resizing the destination accordingly -FluidBufSelectEvery.process(s,b, destination: c, frameHop: 2, chanHop: 3, action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); +~src = Buffer.read(s,FluidFilesPath("Tremblay-Iterative-M.wav")); -//source buffer boundaries still apply before the hopping selection -FluidBufSelectEvery.process(s,b, startFrame: 1, numFrames: 3, startChan: 2, numChans: 3, destination: c, frameHop: 1, chanHop: 2, action: {c.query}); -c.getToFloatArray(action: {|x|x.round(0.1).postln;}); -:: +( +~loudnessbuf = Buffer(s); +~statsbuf = Buffer(s); +~selected = Buffer(s); +FluidBufLoudness.processBlocking(s,~src,features:~loudnessbuf); +FluidBufStats.processBlocking(s,~loudnessbuf,stats:~statsbuf,numChans:1,numDerivs:2); +FluidBufSelectEvery.processBlocking(s,~statsbuf,frameHop:7,destination:~selected); +~selected.loadToFloatArray(action:{ + arg fa; + ["mean loudness", "mean of the 1st derivative of loudness","mean of the 2nd derivative of loudness"].do{ + arg name, i; + "%\t<-- %".format(fa[i],name).postln; + }; +}); +) +:: \ No newline at end of file diff --git a/example-code/sc/BufSines.scd b/example-code/sc/BufSines.scd index e944216a..228ad667 100644 --- a/example-code/sc/BufSines.scd +++ b/example-code/sc/BufSines.scd @@ -1,55 +1,79 @@ - code:: -// create some buffers + ( -b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); -c = Buffer.new(s); -d = Buffer.new(s); +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-BeatRemember.wav"),channels:[0]); +~sines = Buffer(s); +~residual = Buffer(s); ) -// run the process with basic parameters +// listen to the original if you want +~src.play; + +FluidBufSines.processBlocking(s,~src,sines:~sines,residual:~residual,action:{"done".postln;}); + +~sines.play; +~residual.play; + +// change some parameters + +// tracks can be short but the detection threshold is higher than the default ( -Routine{ - t = Main.elapsedTime; - FluidBufSines.process(s, b, sines: c, residual:d).wait; - (Main.elapsedTime - t).postln; -}.play +FluidBufSines.processBlocking(s, + ~src, + sines:~sines, + residual:~residual, + detectionThreshold:-40, + minTrackLen:1, + action:{"done".postln;} +); ) -// listen to each component -c.play; -d.play; +~sines.play; +~residual.play; -//nullsumming tests -{(PlayBuf.ar(1, c)) + (PlayBuf.ar(1,d)) - (PlayBuf.ar(1,b,doneAction:2))}.play -:: +// increase the minimum track length +( +FluidBufSines.processBlocking(s, + ~src, + sines:~sines, + residual:~residual, + detectionThreshold:-40, + minTrackLen:15, + action:{"done".postln;} +); +) -STRONG::A stereo buffer example.:: -CODE:: +~sines.play; +~residual.play; -// load two very different files +// lower the threshold but increase the track length drastically ( -b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); +FluidBufSines.processBlocking(s, + ~src, + sines:~sines, + residual:~residual, + detectionThreshold:-80, + minTrackLen:50, + action:{"done".postln;} +); ) -// composite one on left one on right as test signals -FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b) -b.play +~sines.play; +~residual.play; -// create 2 new buffers as destinations -d = Buffer.new(s); e = Buffer.new(s); - -//run the process on them +// increase the threshold drastically but lower the minimum track length ( -Routine{ - t = Main.elapsedTime; - FluidBufSines.process(s, b, sines: d, residual:e, windowSize: 2048, hopSize: 256, fftSize: 16384).wait; - (Main.elapsedTime - t).postln; -}.play +FluidBufSines.processBlocking(s, + ~src, + sines:~sines, + residual:~residual, + detectionThreshold:-24, + minTrackLen:1, + action:{"done".postln;} +); ) -//listen: stereo preserved! -d.play -e.play -:: +~sines.play; +~residual.play; + +:: \ No newline at end of file diff --git a/example-code/sc/BufSpectralShape.scd b/example-code/sc/BufSpectralShape.scd index 6be48827..d27a6dcb 100644 --- a/example-code/sc/BufSpectralShape.scd +++ b/example-code/sc/BufSpectralShape.scd @@ -1,50 +1,77 @@ - code:: -// create some buffers -( -b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -c = Buffer.new(s); -) +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); + +// listen if you want +~src.play; -// run the process with basic parameters ( -Routine{ - t = Main.elapsedTime; - FluidBufSpectralShape.process(s, b, features: c).wait; - (Main.elapsedTime - t).postln; -}.play -) +var specbuf = Buffer(s); +FluidBufSpectralShape.processBlocking(s,~src,features:specbuf); +specbuf.loadToFloatArray(action:{ + arg fa; + var spec = fa.clump(specbuf.numChannels); + var win; -// listen to the source and look at the buffer -b.play; -c.plot(separately:true) -:: + defer{ + win = Window("FluidBufSpectralShape: click me to hear sines of the spec analysis",Rect(0,0,1600,400)); -STRONG::A stereo buffer example.:: -CODE:: + FluidWaveform(~src,featuresBuffer:specbuf,parent:win,bounds:win.bounds,stackFeatures:true,standalone:false); -// load two very different files -( -b = Buffer.read(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav")); -c = Buffer.read(s,FluidFilesPath("Tremblay-AaS-AcousticStrums-M.wav")); + UserView(win,win.bounds).mouseDownAction_{ + arg view, x, y; + var index = x.linlin(0,view.bounds.width,0,spec.size-1).asInteger; + "analysis frame: %".format(index).postln; + FluidSpectralShape.features.do{ + arg feat, i; + "\t%: %".format(feat.asString.padRight(10),spec[index][i]).postln; + }; + "".postln; + }; + + win.front; + } +}); ) +:: +strong::Using "select":: +By passing an array of symbols to the select argument you can choose to output only a subset of the analyses. The options are \centroid, \spread, \skewness, \kurtosis, \rolloff, \flatness, \and crest. If nothing is specified, the object will return all the analyses. The analyses will always appear in their normal order, this argument just allows for a selection of them to be returned. Reordering the options in this argument will not reorder how the analyses are returned. +code:: -// composite one on left one on right as test signals -FluidBufCompose.process(s, c, numFrames:b.numFrames, startFrame:555000,destStartChan:1, destination:b) -b.play +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -// create a buffer as destinations -c = Buffer.new(s); +// listen if you want +~src.play; + +// change this if you want to see other selected analyses +~selections = [\centroid,\spread,\flatness]; -//run the process on them ( -Routine{ - t = Main.elapsedTime; - FluidBufSpectralShape.process(s, b, features: c).wait; - (Main.elapsedTime - t).postln; -}.play +var specbuf = Buffer(s); +FluidBufSpectralShape.processBlocking(s,~src,features:specbuf,select:~selections); +specbuf.loadToFloatArray(action:{ + arg fa; + var spec = fa.clump(specbuf.numChannels); + var win; + + defer{ + win = Window("FluidBufSpectralShape: click me to hear sines of the spec analysis",Rect(0,0,1600,400)); + + FluidWaveform(~src,featuresBuffer:specbuf,parent:win,bounds:win.bounds,stackFeatures:true,standalone:false); + + UserView(win,win.bounds).mouseDownAction_{ + arg view, x, y; + var index = x.linlin(0,view.bounds.width,0,spec.size-1).asInteger; + "analysis frame: %".format(index).postln; + ~selections.do{ + arg feat, i; + "\t%: %".format(feat.asString.padRight(10),spec[index][i]).postln; + }; + "".postln; + }; + + win.front; + } +}); ) -// look at the buffer: 7shapes for left, then 7 shapes for right -c.plot(separately:true) -:: +:: \ No newline at end of file diff --git a/example-code/sc/BufThreadDemo.scd b/example-code/sc/BufThreadDemo.scd index acb5ff73..036f92b0 100644 --- a/example-code/sc/BufThreadDemo.scd +++ b/example-code/sc/BufThreadDemo.scd @@ -1,9 +1,10 @@ For a thorough explanation, please refer to the tutorial on link::Guides/FluidBufMultiThreading::. +Using code::.process:: will spawn a new thread to do the processing. CODE:: // define a destination buffer -b=Buffer.alloc(s,1); +b = Buffer.alloc(s,1); // a simple call, where we query the destination buffer upon completion with the action message. FluidBufThreadDemo.process(s, b, 1000, action:{|x|x.get(0,{|y|y.postln});}); @@ -13,5 +14,32 @@ c = FluidBufThreadDemo.process(s, b, 100000, action: {|x|x.get(0,{|y|y.postln}); c.cancel; // if a simple call to the UGen is used, the progress can be monitored. The usual cmd-period will cancel the job by freeing the synth. -{c = FluidBufThreadDemo.kr(b,10000).poll; FreeSelfWhenDone.kr(c)}.scope; +( +{ + c = FluidBufThreadDemo.kr(b,10000).poll; + FreeSelfWhenDone.kr(c) +}.scope; +) :: +strong::processBlocking:: +Using code::.processBlocking:: will add the process to the OCS server queue to ensure that the operations get done in order. +code:: + +b = Buffer.alloc(s,1); + +( +FluidBufThreadDemo.processBlocking(s, b, 1000, action:{"job 1 done".postln}); +FluidBufThreadDemo.processBlocking(s, b, 500, action:{"job 2 done".postln}); +FluidBufThreadDemo.processBlocking(s, b, 1500, action:{"job 3 done".postln}); +) + +( +fork{ + FluidBufThreadDemo.processBlocking(s, b, 1000, action:{"job 1 done".postln}); + s.sync; + FluidBufThreadDemo.processBlocking(s, b, 500, action:{"job 2 done".postln}); + s.sync; + FluidBufThreadDemo.processBlocking(s, b, 1500, action:{"job 3 done".postln}); +} +) +:: \ No newline at end of file diff --git a/example-code/sc/Chroma.scd b/example-code/sc/Chroma.scd index 8fa18a43..b36c88b1 100644 --- a/example-code/sc/Chroma.scd +++ b/example-code/sc/Chroma.scd @@ -1,95 +1,65 @@ code:: -//create a monitoring bus for the descriptors -b = Bus.new(\control,0,24); -//create a monitoring window for the values - -( -w = Window("Chroma Bins Monitor", Rect(10, 10, 620, 320)).front; -a = MultiSliderView(w,Rect(10, 10, 600, 300)).elasticMode_(1).isFilled_(1); -) - -//run the window updating routine. -( -~winRange = 0.1; -r = Routine { - { - b.get({ arg val; - { - if(w.isClosed.not) { - a.value = val/~winRange; - } - }.defer - }); - 0.01.wait; - }.loop -}.play -) +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]); -//play a simple sound to observe the values ( -x = { - var source = SinOsc.ar(LFTri.kr(0.1).exprange(80,800),0,0.1); - Out.kr(b,FluidChroma.kr(source, numChroma: 24, maxNumChroma:24)); - source.dup; +~synth = { + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var chroma = FluidChroma.kr(sig,normalize:1); // normalize = 1 makes the output invariant to the signal loudness + var sines; + chroma = FluidStats.kr(chroma,40)[0]; // index 0 to get just the means + SendReply.kr(Impulse.kr(30),"/chroma",chroma); + sines = SinOsc.ar((69..80).midicps,0,chroma * -5.dbamp); + sines = Splay.ar(sines); + sig.dup + sines; }.play; -) - -//we can observe that in the low end, the precision of the fft is not good enough to give a sharp pitch centre. We still can observe the octaviation quantized in quarter tones. -// free this source -x.free +//create a monitoring window for the values -// load a more exciting one -c = Buffer.read(s,FluidFilesPath("Tremblay-SlideChoirAdd-M.wav")); +~win = Window("Chroma Monitor", Rect(10, 10, 620, 320)).front; +~ms = MultiSliderView(~win,Rect(10, 10, 600, 300)).elasticMode_(1).isFilled_(1); -// analyse with parameters to be changed -( -x = {arg chroma = 24, low = 20, high = 20000, norm=1, t_cue, sel=0; - var source = Select.ar(sel, [ - PlayBuf.ar(1,c,loop:1), - Saw.ar(TIRand.kr(60.dup(3),96,t_cue).poll(t_cue).midicps,0.05).sum; - ]); - Out.kr(b,FluidChroma.kr(source ,numChroma: chroma, minFreq: low, maxFreq: high, normalize: norm, maxNumChroma: 24, windowSize: 4096) / 10); - source.dup; -}.play; +OSCdef(\chroma,{ + arg msg; + defer{ + ~ms.value_(msg[3..]); + }; +},"/chroma"); ) -//set the winRange to a more informative value -~winRange = 0.03; - -//instead, let's normalise each frame independently -~winRange = 0.12; -x.set(\norm, 2); - -// observe the number of chroma. The unused ones at the top are not updated -x.set(\chroma,12) - -// back to the full range -x.set(\chroma,24) - -// change the source to random three-note chords -x.set(\sel, 1) +:: +strong::More than 12 pitch classes:: +code:: -// trigger new chords and observe the chroma contour -x.set(\t_cue, 1) +// dividing the octave (and therefore chroma into 19 equal divisions of the octave: -// focus all the chroma bin on a low mid range (there might be nothing!) -x.set(\low,320, \high, 800) +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-SA-UprightPianoPedalWide.wav"),channels:[0]); -// or on a specific octave -x.set(\low, 60.midicps, \high, 72.midicps) +( +~synth = { + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var chroma = FluidChroma.kr(sig,normalize:1,numChroma:19); // normalize = 1 makes the output invariant to the signal loudness + var freqs, sines; + chroma = FluidStats.kr(chroma,40)[0]; // index 0 to get just the means + SendReply.kr(Impulse.kr(30),"/chroma",chroma); + freqs = chroma.numChannels.collect{arg i; 440 * 2.pow(i/chroma.numChannels)}; + sines = SinOsc.ar(freqs,0,chroma * -5.dbamp); + sines = Splay.ar(sines); + sig.dup + sines; +}.play; -// back to full range -x.set(\low,20, \high, 20000) +//create a monitoring window for the values -// free everything -x.free;b.free;c.free;r.stop; -:: +~win = Window("Chroma Monitor", Rect(10, 10, 620, 320)).front; +~ms = MultiSliderView(~win,Rect(10, 10, 600, 300)).elasticMode_(1).isFilled_(1); -STRONG::A musical example:: +OSCdef(\chroma,{ + arg msg; + defer{ + ~ms.value_(msg[3..]); + }; +},"/chroma"); +) -CODE:: -//something will happen here. -:: +:: \ No newline at end of file diff --git a/example-code/sc/DataSet.scd b/example-code/sc/DataSet.scd index c8aee105..747c15cd 100644 --- a/example-code/sc/DataSet.scd +++ b/example-code/sc/DataSet.scd @@ -1,124 +1,254 @@ - CODE:: -// Create a simple a one-dimensional data set, three ways + +( +~ds = FluidDataSet(s); +~databuf = Buffer.loadCollection(s,[0]); +) + +// add a point +( +~ds.addPoint("my-point",~databuf); +~ds.print; +) + +// setPoint adds a point or, if the identifier exists, updates the point +( +fork{ + ~databuf.set(0,99); + ~ds.setPoint("my-point",~databuf); + s.sync; + ~databuf.set(0,87); + ~ds.setPoint("another-point",~databuf); + ~ds.print; +} +) + +// update an existing point +( +~databuf.set(0,236); +~ds.updatePoint("another-point",~databuf); +~ds.print; +) + +// get a point +( +~ds.getPoint("my-point",~databuf); +~databuf.loadToFloatArray(action:{ + arg fa; + fa.postln; +}); +) + +// delete a point +( +~ds.deletePoint("my-point"); +~ds.print; +) + +:: +strong::Adding an audio analysis to a DataSet:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); + +// add just the mean mfcc values to the dataset +( +~mfccs = Buffer(s); +~stats = Buffer(s); +~flat = Buffer(s); +FluidBufMFCC.processBlocking(s,~src,features:~mfccs); +FluidBufStats.processBlocking(s,~mfccs,stats:~stats,select:[\mean]); +FluidBufFlatten.processBlocking(s,~stats,destination:~flat); +FluidDataSet(s).addPoint("ASWINE",~flat).print; +) + +// use .fromBuffer to add *every* mfcc analysis to the dataset +( +FluidBufMFCC.processBlocking(s,~src,features:~mfccs); +FluidDataSet(s).fromBuffer(~mfccs).print; +) + +:: +strong::Four ways to get a bunch of data into a dataset:: +code:: // Using routine -s.reboot; ( fork{ - ~ds = FluidDataSet.new(s); - ~point = Buffer.alloc(s,1,1); + var point = Buffer.alloc(s,1); + var ds = FluidDataSet(s); s.sync; - 10.do{|i| - ~point.set(0,i); - ~ds.addPoint(i.asString,~point,{("addPoint"+i).postln}); - s.sync; + 10.do{ + arg i; + point.set(0,i); + ds.addPoint("point-%".format(i),point); + // s.sync; }; - ~ds.dump; - s.sync; - ~ds.free; + ds.print; }; ) //Using Dictionary ( -d = Dictionary.new; -d.add(\cols -> 1); -d.add(\data -> Dictionary.newFrom(10.collect{|i|[i.asString, [i.asFloat]]}.flatten)); -fork{ - ~ds = FluidDataSet.new(s); - ~ds.load(d); s.sync; - ~ds.dump; s.sync; ~ds.free; -} +var dict = Dictionary.new; +dict.put(\cols,1); +dict.put(\data, + Dictionary.newFrom( + [ // one could, of course, make this array more programmatically + "point-0",0, + "point-1",1, + "point-2",2, + "point-3",3, + "point-4",4, + "point-5",5, + "point-6",6, + "point-7",7, + "point-8",8, + "point-9",9 + ] + ) +); +FluidDataSet(s).load(dict).print; ) // Using a synth ( -~ds = FluidDataSet.new(s); +var ds = FluidDataSet.new(s); { var trig = Impulse.kr(20); var count = PulseCount.kr(trig) - 1; var buf = LocalBuf(1); BufWr.kr(count, buf); - FluidDataSetWr.kr(~ds.asUGenInput, idNumber: count, buf: buf, trig: trig); + FluidDataSetWr.kr(ds, idNumber: count, buf: buf, trig: trig); FreeSelf.kr(count - 8); -}.play.onFree{~ds.dump{|o| o.postln;~ds.free}} +}.play.onFree{ds.print} +) + +// from a buffer +( +fork{ + var buf = Buffer.loadCollection(s,(0..9)); + s.sync; + FluidDataSet(s).fromBuffer(buf).print; +} ) :: +strong::More Messages:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); -STRONG:: Buffer Interface:: +// use .fromBuffer to add *every* mfcc analysis to the dataset +( +~mfccs = Buffer(s); +FluidBufMFCC.processBlocking(s,~src,features:~mfccs); +~ds = FluidDataSet(s).fromBuffer(~mfccs).print; +) -As the content of the dataset has a similar structure to buffers, namely arrays of floats in parallel, it is possible to transfer the content between the two. Careful consideration of the rotation of the buffer, as well as the relation of points to channel numbers, are needed. +// write to disk +~ds.write(Platform.defaultTempDir+/+"temp_dataset.json"); -code:: +// read from disk ( -//Make a dummy data set -d = FluidDataSet(s); -~data = Dictionary.with(*Array.iota(20).reshape(4,5).collect{|a,i| ("row"++i)->a}); -~dsdata = Dictionary.newFrom([\cols,5,\data,~data]); -d.load(~dsdata); -d.print; -) - -//convert to separate buffer and labelset -b = Buffer(s); -l = FluidLabelSet(s); -d.toBuffer(b,0,l); - -//check the result: by default, dataset points become frames, with their associated data columns as channels -b.query -b.getn(0,20,{|x|x.postln}) -l.print - -//you can also transpose your query, where dataset points are each a buffer channel, and each data column becomes a buffer frame -d.toBuffer(b,1,l); -b.query -b.getn(0,20,{|x|x.postln}) -//note that the IDs are still one per item, as columns are unamed in datasets -l.print - -//Convert back to DS again -e = FluidDataSet(s); - -//Let's use the transposed data we just got -e.print; -e.fromBuffer(b,1,l); -e.print; -//if we didn't transpose, we would get an error as the labelset is mismatched with the number of items -e.clear -e.print -e.fromBuffer(b,0,l) -:: +~loaded_ds = FluidDataSet(s).read(Platform.defaultTempDir+/+"temp_dataset.json"); +~loaded_ds.print; +) + +// how many data points are there +~ds.size + +// how many dimensions +~ds.cols + +// dump the contents to a language side dict +( +~ds.dump({ + arg dict; + dict["data"].keysValuesDo{ + arg k, v; + "%:\t%".format(k,v).postln; + }; + "this data set has % dimensions".format(dict["cols"]).postln; +}); +) -STRONG:: Merging Datasets:: +// clear it +( +~ds.clear; +~ds.print; +) +:: +strong::Merge two data sets that have the same number of dimensions:: code:: -//this is how to add items between 2 datasets. -//create 2 datasets + ( -~dsA = FluidDataSet.new(s); -~dsB = FluidDataSet.new(s); +~srcA = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); +~srcB = Buffer.read(s,FluidFilesPath("Harker-DS-TenOboeMultiphonics-M.wav")); ) -//feed them items with same dimensions but different identifiers -~dsA.load(Dictionary.newFrom([\cols, 1, \data, Dictionary.newFrom([\one,1,\two,2])])); -~dsB.load(Dictionary.newFrom([\cols, 1, \data, Dictionary.newFrom([\three,3,\four,4])])); -~dsA.print; -~dsB.print; +// use .fromBuffer to add *every* mfcc analysis to the dataset +( +fork{ + var mfccs = Buffer(s); + var flat = Buffer(s); -// merge and check. it works. -~dsB.merge(~dsA) -~dsB.print; + FluidBufMFCC.processBlocking(s,~srcA,features:mfccs); + s.sync; -//change the content of the dataset to shared identifiers -~dsA.load(Dictionary.newFrom([\cols, 1, \data, Dictionary.newFrom([\three,333,\four,444])])); -~dsB.load(Dictionary.newFrom([\cols, 1, \data, Dictionary.newFrom([\three,3,\four,4])])); -~dsA.print; -~dsB.print; + ~dsA = FluidDataSet(s); + mfccs.numFrames.do{ + arg i; + FluidBufFlatten.processBlocking(s,mfccs,i,1,destination:flat); + ~dsA.addPoint("sourceA-%".format(i),flat); + }; + + + FluidBufMFCC.processBlocking(s,~srcB,features:mfccs); + s.sync; -//try to merge, it does not update -~dsB.merge(~dsA) -~dsB.print; + ~dsB = FluidDataSet(s); + mfccs.numFrames.do{ + arg i; + FluidBufFlatten.processBlocking(s,mfccs,i,1,destination:flat); + ~dsB.addPoint("sourceB-%".format(i),flat); + }; -// add the overwrite flag, and it works -~dsB.merge(~dsA,1) -~dsB.print; + ~dsA.print; + ~dsB.print; +} +) + +// merge dataset B into dataset A +// because there are no overlapping identifiers, +// nothign will be overwritten +( +~dsA.merge(~dsB); +~dsA.print; +) :: +strong::Using:: code::.fromBuffer:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); + +// use .fromBuffer to add *every* mfcc analysis to the dataset +// pass a labelset so the identifiers aren't just numbers +( +fork{ + var mfccs = Buffer(s); + FluidBufMFCC.processBlocking(s,~src,features:mfccs); + s.sync; + ~ls = FluidLabelSet(s); + mfccs.numFrames.do{ + arg i; + // in the label set that gets used to create + // the identifiers, the identifier is the + // frame number in the buffer (an integer) + // and the "label" is what will be come the + // identifier in the dataset + ~ls.addLabel(i,"ASWINE-mfcc-%".format(i)); + }; + ~ds = FluidDataSet(s).fromBuffer(mfccs,0,~ls).print; +} +) + +:: \ No newline at end of file diff --git a/example-code/sc/DataSetQuery.scd b/example-code/sc/DataSetQuery.scd index 208b007d..43373141 100644 --- a/example-code/sc/DataSetQuery.scd +++ b/example-code/sc/DataSetQuery.scd @@ -1,79 +1,177 @@ - code:: -s.reboot; // Create a DataSet with known data -~dataSet= FluidDataSet(s); ( -~points = 100.collect{|i|5.collect{|j|j+(i/100)}}; -~dataSet.clear; -~tmpbuf = Buffer.alloc(s,5); +var dict = Dictionary.newFrom([ + "cols",5, + "data",Dictionary.newFrom( + 100.collect{ + arg i; + var point = 5.collect{ + arg j; + j+(i/100); + }; + [i,point]; + }.flatten + ) +]); + +//the integer part of the value is the dimension, and the fractional part is the identifier. +~ds = FluidDataSet(s).load(dict).print; -fork{ - s.sync; - ~points.do{|x,i| - ~tmpbuf.setn(0,x); - ~dataSet.addPoint(i,~tmpbuf); - s.sync; - } -} +~tmpbuf = Buffer.alloc(s,5); ) -//check the source - the column is the integer part of the value, and the row is the fractional part. This will help us identify what we kept in our didactic query -~dataSet.print; // Prepare a FluidDataSetQuery object -~query = FluidDataSetQuery.new; +( +~query = FluidDataSetQuery(s); ~out = FluidDataSet(s); // prepare a simple query +// select points where the value in column 0 is less than 0.04 ~query.filter(0,"<",0.04); -~query.addColumn(2); -~query.transform(~dataSet, ~out); +~query.addColumn(2); // the column we actually want copied into ~out is column 2 +~query.transform(~ds, ~out); // check the result ~out.print; +) +( //prepare a more complex query ~query.clear; ~query.filter(0,">",0.03); ~query.and(1,"<",1.08); ~query.or(2,">",2.98); -~query.addRange(2,2); -~query.transform(~dataSet, ~out); +~query.addRange(2,3); // addRange will add 3 columns starting at column index 2 +~query.transform(~ds, ~out); // Check the results ~out.print; +) :: - STRONG:: Joining Datasets:: - code:: -//this is how to join 2 datasets, adding columns to items with the same identifier -//create 3 datasets + +// this is how to join 2 datasets, adding columns to items with the same identifier +// create 2 datasets + ( -~dsA = FluidDataSet(s); -~dsB = FluidDataSet(s); -~dsC = FluidDataSet(s); +~dsA = FluidDataSet(s); +~dsB = FluidDataSet(s); ) //feed them items with almost overlaping identifier lists but with different dimensions -~dsA.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\zero, [0,0], \one,[1,11],\two,[2,22], \three,[3,33],\four,[4,44]])])); -~dsB.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\one,[111,1111],\two,[222,2222], \three,[333,3333],\four,[444,4444],\five,[555,5555]])])); +( +~dsA.load( + Dictionary.newFrom([ + \cols, 2, + \data, Dictionary.newFrom([ + \zero, [0,0], + \one,[1,11], + \two,[2,22], + \three,[3,33], + \four,[4,44] + ]) + ]) +); + +~dsB.load +(Dictionary.newFrom([ + \cols, 2, + \data, Dictionary.newFrom([ + \one,[111,1111], + \two,[222,2222], + \three,[333,3333], + \four,[444,4444], + \five,[555,5555] + ]) +]) +); + ~dsA.print; ~dsB.print; +) // no query/filter defined, copies all items with identifiers common to both, and all of the defined column of the first input -~joiner = FluidDataSetQuery.new; -~joiner.transformJoin(~dsA,~dsB,~dsC) -~dsC.print - -// all the sophisticated conditions applicable to 'transform' can be done on the first dataset too. Selected columns of the first source are appended to matching items in the second source. -~joiner.filter(0,">",2.1) -~joiner.and(1,"<", 40) -~joiner.addColumn(0) -~joiner.transformJoin(~dsA,~dsB,~dsC) -~dsC.print +( +~ds_joined = FluidDataSet(s); +~joiner = FluidDataSetQuery(s); +~joiner.transformJoin(~dsA,~dsB,~ds_joined); +~ds_joined.print +) + +( +// all the conditions applicable to 'transform' can be done on the first dataset too. Selected columns of the first source are appended to matching items in the second source. +~joiner.clear; +~joiner.filter(0,">",2.1); +~joiner.and(1,"<", 40); +~joiner.addColumn(0); +~joiner.transformJoin(~dsA,~dsB,~ds_joined); +~ds_joined.print; +) + :: +strong::Audio Analysis Data:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); + +// take a listen +~src.play; + +// make a dataset of pitch anlyses +( +~features = Buffer(s); +FluidBufPitch.processBlocking(s,~src,features:~features); +~ds_pitch = FluidDataSet(s).fromBuffer(~features); +~ds_pitch.print; +) + +// select only the pitch analyses where the pitch confidence is > 0.9 +( +~ds_filtered = FluidDataSet(s); +~dsq = FluidDataSetQuery(s); +~dsq.filter(1,">",0.9); +~dsq.addColumn(0); +~dsq.transform(~ds_pitch,~ds_filtered); +~ds_filtered.print; +) + +// make another dataset of spectral analyses +( +FluidBufSpectralShape.processBlocking(s,~src,features:~features); +~ds_spec = FluidDataSet(s).fromBuffer(~features); +~ds_spec.print; +) + +// select only the spectral analyses where the pitch confidence is > 0.9; +( +~ds_selected = FluidDataSet(s); +~dsq.clear; +~dsq.filter(1,">",0.9); +~dsq.transformJoin(~ds_pitch,~ds_spec,~ds_selected); +~ds_selected.print; +) + +// or just append it all together to make one big dataset +( +~dsq.clear; +~dsq.addRange(0,2); // starting at column 0 add two columns (both of the columns in ~ds_pitch) +~dsq.transformJoin(~ds_pitch,~ds_spec,~ds_selected); +~ds_selected.print; +) + +// or do it all at once: +( +~dsq.clear; +~dsq.filter(1,">",0.9); // only analyses where the pitch conf is high +~dsq.addColumn(0); // we'll take the pitch freq... +~dsq.transformJoin(~ds_pitch,~ds_spec,~ds_selected);// ... and append it to the spectral analyses +~ds_selected.print; +) + +:: \ No newline at end of file diff --git a/example-code/sc/Grid.scd b/example-code/sc/Grid.scd index 2d321ca7..56450790 100644 --- a/example-code/sc/Grid.scd +++ b/example-code/sc/Grid.scd @@ -1,158 +1,263 @@ - -STRONG::A didactic example:: - code:: -/// make a simple grid of numbers -~simple = Dictionary.newFrom(36.collect{|i|[i.asSymbol, [i.mod(9), i.div(9)]]}.flatten(1)); - -//look at it ( -w = Window("the source", Rect(128, 64, 230, 100)); -w.drawFunc = { - Pen.use { - ~simple.keysValuesDo{|key, val| - Pen.stringCenteredIn(key, Rect((val[0] * 25), (val[1] * 25), 25, 25), Font( "Helvetica", 12 ), Color.black) - } - } -}; -w.refresh; -w.front; -) +// peek at a didactic dataset (color is just to track better where the points end up) +~ds = FluidDataSet(s).read(FluidFilesPath("../Data/moon.json")); +~colors = Dictionary.new; +~ds.dump({ + arg dict; + dict["data"].keysValuesDo{ + arg k, v; + var hsl = Color.hsv(v[0],1,0.5); + ~colors[k] = Color(v[0],v[1],hsl.red); + }; + defer{ + var fp = FluidPlotter(dict:dict); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); + }; + fp.pointSizeScale_(3); + }; +}); +) -//load it in a dataset -~raw = FluidDataSet(s); -~raw.load(Dictionary.newFrom([\cols, 2, \data, ~simple])); +// now gridify it +( +~ds_gridded = FluidDataSet(s); +~grid = FluidGrid(s).fitTransform(~ds,~ds_gridded); -// make a grid out of it -~grid = FluidGrid(s); -~gridified = FluidDataSet(s); -~grid.fitTransform(~raw, ~gridified, action:{~gridified.dump{|x| ~gridifiedDict = x["data"]; \gridded.postln;}}) +// by default it is not normalized, so we'll do that before +// sending it to plotter +FluidNormalize(s).fitTransform(~ds_gridded,~ds_gridded); -// watch the grid -( -w = Window("a perspective", Rect(358, 64, 350, 230)); -w.drawFunc = { - Pen.use { - ~gridifiedDict.keysValuesDo{|key, val| - Pen.stringCenteredIn(key, Rect((val[0] * 25), (val[1] * 25), 25, 25), Font( "Helvetica", 12 ), Color.black) - } - } -}; -w.refresh; -w.front; +~ds.dump({ + arg original; + ~ds_gridded.dump({ + arg gridded; + defer{ + var win = Window(bounds:Rect(0,0,1000,500)); + var fps = [original,gridded].collect{ + arg dict; + var fp = FluidPlotter(dict:dict,standalone:false); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); + }; + fp.pointSizeScale_(3); + fp; + }; + win.layout_(HLayout(*fps)); + win.front; + }; + }); +}); ) +:: +strong::Oversampling:: +code:: -// change the constraints and draw again +// if we added an oversampling of 4 to the example above, +// there will be 4x as many points on the grid as there are +// in the dataset. they don't all get used, so we can still +// see some of the original shape ( -~grid.axis_(0).extent_(4).fitTransform(~raw, ~gridified, action:{ - ~gridified.dump{|x| - ~gridifiedDict = x["data"];\gridded.postln; - {w.refresh;}.defer; -}}) -) +~ds_gridded = FluidDataSet(s); +~grid = FluidGrid(s,4).fitTransform(~ds,~ds_gridded); -// here we constrain in the other dimension -( -~grid.axis_(1).extent_(3).fitTransform(~raw, ~gridified, action:{ - ~gridified.dump{|x| - ~gridifiedDict = x["data"];\gridded.postln; - {w.refresh;}.defer; -}}) +// by default it is not normalized, so we'll do that before +// sending it to plotter +FluidNormalize(s).fitTransform(~ds_gridded,~ds_gridded); + +~ds.dump({ + arg original; + ~ds_gridded.dump({ + arg gridded; + defer{ + var win = Window(bounds:Rect(0,0,1000,500)); + var fps = [original,gridded].collect{ + arg dict; + var fp = FluidPlotter(dict:dict,standalone:false); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); + }; + fp.pointSizeScale_(3); + fp; + }; + win.layout_(HLayout(*fps)); + win.front; + }; + }); +}); ) :: - -STRONG::A more colourful example exploring oversampling:: - +strong::Axis and Extent:: code:: -// make all dependencies +// by adding an extent of 5 to axis 0, we get 5 columns: ( -~raw = FluidDataSet(s); -~standardized = FluidDataSet(s); -~reduced = FluidDataSet(s); -~normalized = FluidDataSet(s); -~standardizer = FluidStandardize(s); -~normalizer = FluidNormalize(s, 0.05, 0.95); -~umap = FluidUMAP(s).numDimensions_(2).numNeighbours_(5).minDist_(0.2).iterations_(50).learnRate_(0.2); -~grid = FluidGrid(s); -~gridified = FluidDataSet(s); -) +~ds_gridded = FluidDataSet(s); +~grid = FluidGrid(s,oversample:1,extent:5,axis:0).fitTransform(~ds,~ds_gridded); -// build a dataset of 400 points in 3D (colour in RGB) -~colours = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1)); -~raw.load(Dictionary.newFrom([\cols, 3, \data, ~colours])); +// by default it is not normalized, so we'll do that before +// sending it to plotter +FluidNormalize(s).fitTransform(~ds_gridded,~ds_gridded); -//First standardize our DataSet, then apply the UMAP to get 2 dimensions, then normalise these 2 for drawing in the full window size -( -~standardizer.fitTransform(~raw,~standardized,action:{"Standardized".postln}); -~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln}); -~normalizer.fitTransform(~reduced,~normalized,action:{"Normalized Output".postln}); +~ds.dump({ + arg original; + ~ds_gridded.dump({ + arg gridded; + defer{ + var win = Window(bounds:Rect(0,0,1000,500)); + var fps = [original,gridded].collect{ + arg dict; + var fp = FluidPlotter(dict:dict,standalone:false); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); + }; + fp.pointSizeScale_(3); + fp; + }; + win.layout_(HLayout(*fps)); + win.front; + }; + }); +}); ) -//we recover the reduced dataset -~normalized.dump{|x| ~normalizedDict = x["data"]}; -//Visualise the 2D projection of our original 4D data + +// different settings ( -w = Window("a perspective", Rect(128, 64, 200, 200)); -w.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours[key.asSymbol].flat; - } - } -}; -w.refresh; -w.front; +~ds_gridded = FluidDataSet(s); +~grid = FluidGrid(s,oversample:1,extent:7,axis:1).fitTransform(~ds,~ds_gridded); + +// by default it is not normalized, so we'll do that before +// sending it to plotter +FluidNormalize(s).fitTransform(~ds_gridded,~ds_gridded); + +~ds.dump({ + arg original; + ~ds_gridded.dump({ + arg gridded; + defer{ + var win = Window(bounds:Rect(0,0,1000,500)); + var fps = [original,gridded].collect{ + arg dict; + var fp = FluidPlotter(dict:dict,standalone:false); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); + }; + fp.pointSizeScale_(3); + fp; + }; + win.layout_(HLayout(*fps)); + win.front; + }; + }); +}); ) -//Force the UMAP-reduced dataset into a grid, normalise for viewing then print in another window +// this can also be over sampled: ( -~grid.fitTransform(~reduced,~gridified,action:{"Gridded Output".postln; - ~normalizer.fitTransform(~gridified,~normalized,action:{"Normalized Output".postln; - ~normalized.dump{|x| - ~normalizedDict = x["data"]; - { - y = Window("a grid", Rect(328, 64, 200, 200)); - y.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours[key.asSymbol].flat; - } - } +~ds_gridded = FluidDataSet(s); +~grid = FluidGrid(s,oversample:12,extent:6,axis:0).fitTransform(~ds,~ds_gridded); + +// by default it is not normalized, so we'll do that before +// sending it to plotter +FluidNormalize(s).fitTransform(~ds_gridded,~ds_gridded); + +~ds.dump({ + arg original; + ~ds_gridded.dump({ + arg gridded; + defer{ + var win = Window(bounds:Rect(0,0,1000,500)); + var fps = [original,gridded].collect{ + arg dict; + var fp = FluidPlotter(dict:dict,standalone:false); + ~colors.keysValuesDo{ + arg k, v; + fp.pointColor_(k,v); }; - y.refresh; - y.front; - }.defer; + fp.pointSizeScale_(3); + fp; + }; + win.layout_(HLayout(*fps)); + win.front; }; }); }); ) +:: +strong::With Sound:: +Using link::Classes/FluidUMAP:: to dimensionally reduce 13 dimensions of link::Classes/FluidBufMFCC:: analyses down to two dimensions, play with each of the plots and see which is more musically expressive. Each has it's strenghts and weaknesses! +code:: -// We can check the dimensions of the yielded grid by dumping the normalisation.The grid coordinates are zero-counting -~normalizer.dump{|x|x["data_max"].postln} +~src = Buffer.readChannel(s,FluidFilesPath("Green-Box641.wav"),channels:[0]); -// This looks ok, but let's improve it with oversampling +// make a dataset of MFCC analyses all 100 ms long, the id is the sample it starts at ( -~grid.oversample_(3).fitTransform(~reduced,~gridified,action:{"Gridded Output".postln; - ~normalizer.fitTransform(~gridified,~normalized,action:{"Normalized Output".postln; - ~normalized.dump{|x| - ~normalizedDict = x["data"]; - { - y.refresh; - }.defer; - }; - }); +var mfccs = Buffer(s); +var stats = Buffer(s); +var flat = Buffer(s); +var currentSample = 0; +var hopSamples = ~src.sampleRate * 0.1; +~ds = FluidDataSet(s); +while({ + (currentSample + hopSamples) < ~src.numFrames; +},{ + FluidBufMFCC.processBlocking(s,~src,currentSample,hopSamples,startCoeff:1,features:mfccs); + FluidBufStats.processBlocking(s,mfccs,stats:stats,select:[\mid]); + FluidBufFlatten.processBlocking(s,stats,destination:flat); + ~ds.addPoint(currentSample.asInteger,flat); + currentSample = currentSample + hopSamples; }); +~ds.print; ) -// Again, checking the normalisation dump to check the maxima of each dimension -~normalizer.dump{|x|x["data_max"].postln} +// do the data stuff +( +~ds_umap = FluidDataSet(s); +FluidUMAP(s,2,15,0.5).fitTransform(~ds,~ds_umap); // reduce 13 dimensions to 2 dimensions +FluidNormalize(s).fitTransform(~ds_umap,~ds_umap); // normalize it so it plots nicely +~ds_grid = FluidDataSet(s); +FluidGrid(s).fitTransform(~ds_umap,~ds_grid); // gridify it into another dataset +FluidNormalize(s).fitTransform(~ds_grid,~ds_grid); // normalize that too so it plots nicely +~win = Window(bounds:Rect(0,0,1000,500)); +[~ds_umap,~ds_grid].do{ + arg ds, i; + ds.dump({ + arg dict; + var buf = Buffer.alloc(s,2); + var tree = FluidKDTree(s).fit(ds); + var last = nil; + defer{ + FluidPlotter(~win,Rect((i*500) + 10,10,480,480),dict,{ + arg view, x, y; + buf.setn(0,[x,y]); + tree.kNearest(buf,1,{ + arg id; + if(id != last){ + last = id; + view.highlight_(id); + { + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),1,id.asInteger); + var env = EnvGen.kr(Env([0,1,1,0],[0.02,0.06,0.02]),doneAction:2); + sig.dup * env; + }.play; + }; + }); + },standalone:false); + } + }); +}; +~win.front; +) -:: + +:: \ No newline at end of file diff --git a/example-code/sc/KDTree.scd b/example-code/sc/KDTree.scd index e29e857d..ec251534 100644 --- a/example-code/sc/KDTree.scd +++ b/example-code/sc/KDTree.scd @@ -1,69 +1,83 @@ - +strong::Big Example:: code:: -// Make a DataSet of random 2D points -s.reboot; -( -fork{ - d = Dictionary.with( - *[\cols -> 2,\data -> Dictionary.newFrom( - 100.collect{|i| [i, [ 1.0.linrand,1.0.linrand]]}.flatten)]); - ~ds = FluidDataSet(s); - ~ds.load(d, {~ds.print}); -} -) - -// Make a new tree, and fit it to the DataSet -~tree = FluidKDTree(s,numNeighbours:5); - -//Fit it to the DataSet -~tree.fit(~ds); -// Should be 100 points, 2 columns -~tree.size; -~tree.cols; - -//Return the labels of k nearest points to a new point +// using a KDTree to lookup the point on the plot that is nearest to the mouse ( -~p = [ 1.0.linrand,1.0.linrand ]; -~tmpbuf = Buffer.loadCollection(s, ~p, 1, { - ~tree.kNearest(~tmpbuf, action:{ |a|a.postln;~nearest = a;}) +var ds = FluidDataSet(s).read(FluidFilesPath("../Data/flucoma_corpus.json")); +var tree = FluidKDTree(s).fit(ds); +ds.dump({ + arg dict; + var xybuf = Buffer.alloc(s,2); + var slicePoints = Buffer.read(s,FluidFilesPath("../Data/flucoma_corpus_slices.wav")); + var loader = FluidLoadFolder(FluidFilesPath()); + loader.play(s,{ + defer{ + FluidPlotter(dict:dict,mouseMoveAction:{ + arg view, x, y; + xybuf.setn(0,[x,y]); + tree.kNearest(xybuf,1,{ + arg id; + var index = id.asInteger; + defer{ + view.highlight_(id); + }; + + { + var start = Index.kr(slicePoints,index); + var end = Index.kr(slicePoints,index+1); + var sig = PlayBuf.ar(2,loader.buffer,BufRateScale.ir(loader.buffer),1,start)[0]; + var dur_sec = min((end-start) / SampleRate.ir,1); + var env = EnvGen.kr(Env([0,1,1,0],[0.03,dur_sec-0.06,0.03]),doneAction:2); + sig.dup * env; + }.play; + }); + }); + }; + }); }); ) -// Retrieve values from the DataSet by iterating through the returned labels -( -fork{ - ~nearest.do{|n| - ~ds.getPoint(n, ~tmpbuf, {~tmpbuf.getn(0, 2, {|x|x.postln})}); - s.sync; - } -} -) -// Distances of the nearest points -~tree.kNearestDist(~tmpbuf, action:{ |a| a.postln }); - -// Explore changing the number of neighbourgs -~tree.numNeighbours = 11; // note that this value needs to be sent to the server -~tree.kNearest(~tmpbuf, action:{ |a|a.postln;}); -~tree.numNeighbours = 0; // 0 will return all items in order of distance -~tree.kNearest(~tmpbuf, action:{ |a|a.postln;}); - -// Limit the search to an acceptable distance in a radius -// Define a point, and observe typical distance values -~p = [ 0.4,0.4]; +:: +strong::radius and num neighbours:: +code:: + +// set some initial values ( -~tmpbuf = Buffer.loadCollection(s, ~p, 1, { - ~tree.kNearest(~tmpbuf, action:{ |a|a.postln;}); - ~tree.kNearestDist(~tmpbuf, action:{ |a|a.postln;}); -}); +~numNeighbours = 3; +~tree.radius_(0.04); ) -// enter a valid radius. -~tree.radius = 0.1; -// FluidKDTree will return only values that are within that radius, up to numNeighbours values + +// then make the plot, once it's up and you're clicking around, +// change the numbers and re-run the code above to see the differences ( -~tmpbuf = Buffer.loadCollection(s, ~p, 1, { - ~tree.kNearest(~tmpbuf, action:{ |a|a.postln;}); +var ds = FluidDataSet(s).load( + Dictionary.newFrom([ + "cols",2, + "data",Dictionary.newFrom( + 100.collect{ + arg i; + [i,{rrand(0.0,1.0)}!2] + }.flatten + ) + ]) +); +~tree = FluidKDTree(s).fit(ds); +ds.dump({ + arg dict; + var xybuf = Buffer.alloc(s,2); + defer{ + FluidPlotter(dict:dict,mouseMoveAction:{ + arg view, x, y; + xybuf.setn(0,[x,y]); + ~tree.kNearest(xybuf,~numNeighbours,{ + arg id; + defer{ + view.highlight_(id); + }; + }); + }); + }; }); ) :: @@ -82,48 +96,78 @@ For instance, whilst fitting the tree against some n-dimensional descriptor data code:: -( -Routine{ - var inputBuffer = Buffer.alloc(s,2); - var outputBuffer = Buffer.alloc(s,10);//5 neighbours * 2D data points - s.sync; - { - var trig = Impulse.kr(4); //can go as fast as ControlRate.ir/2 - var point = 2.collect{TRand.kr(0,1,trig)}; - point.collect{|p,i| BufWr.kr([p],inputBuffer,i)}; - ~tree.kr(trig,inputBuffer,outputBuffer,5,nil); - Poll.kr(trig, BufRd.kr(1,outputBuffer,Array.iota(10)),10.collect{|i| "Neighbour" + (i/2).asInteger ++ "-" ++ (i.mod(2))}); - Silent.ar; - }.play; -}.play; -) +~src = Buffer.read(s,FluidFilesPath("Constanzo-PreparedSnare-M.wav")); -//Using a lookup data set instead: -//here we populate with numbers that are in effect the indicies, but it could be anything numerical that will be returned on the server-side and would be usable on that side +// create two datasets: +// one of mfcc analyses for each slice and one of the playback information for each slice ( -fork{ - d = Dictionary.with( - *[\cols -> 1,\data -> Dictionary.newFrom( - 100.collect{|i| [i, [ i ]]}.flatten)]); - ~dsL = FluidDataSet.new(s); - ~dsL.load(d, {~dsL.print}); -} +var indices = Buffer(s); +var mfccs = Buffer(s); +var stats = Buffer(s); +var flat = Buffer(s); +var playback_info_dict = Dictionary.newFrom([ + "cols",2, + "data",Dictionary.new; +]); + +~ds_mfccs = FluidDataSet(s); + +FluidBufOnsetSlice.processBlocking(s,~src,indices:indices,metric:9,threshold:0.7); +indices.loadToFloatArray(action:{ + arg fa; + + // go through each slice (from one slice point to the next) + fa.doAdjacentPairs{ + arg start, end, i; + var num = end - start; + var id = "slice-%".format(i); + + // add playback info for this slice to this dict + playback_info_dict["data"][id] = [start,num]; + + FluidBufMFCC.processBlocking(s,~src,start,num,startCoeff:1,features:mfccs); + FluidBufStats.processBlocking(s,mfccs,stats:stats,select:[\mean]); + FluidBufFlatten.processBlocking(s,stats,destination:flat); + + // add analysis info for this slice to this data set + ~ds_mfccs.addPoint(id,flat); + }; + + ~ds_playback = FluidDataSet(s).load(playback_info_dict); + + ~ds_mfccs.print; + ~ds_playback.print; +}); ) -// Create the buffers, and make a synth, querying our tree with some random points +// we'll use this kdtree to find the nearest neighbour in 13 dimensions (mfccs) +~tree = FluidKDTree(s); + +~tree.fit(~ds_mfccs); + +// of course it will often find "itself" because the source and lookup sounds are the same. +// try with different source and lookup sounds ( -Routine{ - var inputBuffer = Buffer.alloc(s,2); - var outputBuffer = Buffer.alloc(s,5);//5 neighbours * 1D points - s.sync; - { - var trig = Impulse.kr(4); //can go as fast as ControlRate.ir/2 - var point = 2.collect{TRand.kr(0,1,trig)}; - point.collect{|p,i| BufWr.kr([p],inputBuffer,i)}; - ~tree.kr(trig,inputBuffer,outputBuffer,5,~dsL); - Poll.kr(trig, BufRd.kr(1,outputBuffer,Array.iota(5)),5.collect{|i| "Neighbour" + i}); - Silent.ar; - }.play; +{ + var src = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var mfccs = FluidMFCC.kr(src,startCoeff:1); + var mfccbuf = LocalBuf(mfccs.numChannels); + var playbackinfo = LocalBuf(2); + var trig = Impulse.kr(10); // could change how often the lookup happens... + var start, num, sig_looked_up; + + FluidKrToBuf.kr(mfccs,mfccbuf); + + // kdtree finding the nearest neighbour in 13 dimensions + ~tree.kr(trig,mfccbuf,playbackinfo,1,~ds_playback); + # start, num = FluidBufToKr.kr(playbackinfo); + + start.poll(label:"start frame"); + num.poll(label:"num frames"); + + // not using num frames for playback here, but one certainly could! + sig_looked_up = PlayBuf.ar(1,~src,BufRateScale.ir(~src),trig,start); + [src,sig_looked_up * -7.dbamp]; }.play; ) :: diff --git a/example-code/sc/KMeans.scd b/example-code/sc/KMeans.scd index e59e4d6e..9d03c999 100644 --- a/example-code/sc/KMeans.scd +++ b/example-code/sc/KMeans.scd @@ -1,154 +1,202 @@ code:: ( -//Make some clumped 2D points and place into a DataSet -~points = (4.collect{ - 64.collect{(1.sum3rand) + [1,-1].choose}.clump(2) - }).flatten(1) * 0.5; -fork{ - ~dataSet = FluidDataSet(s); - d = Dictionary.with( - *[\cols -> 2,\data -> Dictionary.newFrom( - ~points.collect{|x, i| [i, x]}.flatten)]); - s.sync; - ~dataSet.load(d, {~dataSet.print}); -} +// peek at a didactic dataset +~ds = FluidDataSet(s).read(FluidFilesPath("../Data/moon.json")); +~ds.dump({ + arg dict; + defer{ + ~fp = FluidPlotter(dict:dict).pointSizeScale_(3); + }; +}); ) - -// Create a KMeans instance and a LabelSet for the cluster labels in the server -~clusters = FluidLabelSet(s); +// now keeping that window open, try some different numbers for k +( ~kmeans = FluidKMeans(s); +~ls = FluidLabelSet(s); +) -// Fit into 4 clusters +// k = the default of 4 ( -~kmeans.fitPredict(~dataSet,~clusters,action: {|c| - "Fitted.\n # Points in each cluster:".postln; - c.do{|x,i| - ("Cluster" + i + "->" + x.asInteger + "points").postln; - } +~kmeans.fitPredict(~ds,~ls,{ + ~ls.dump({ + arg dict; + ~fp.categories_(dict); }); +}); ) -// Cols of kmeans should match DataSet, size is the number of clusters - -~kmeans.cols; -~kmeans.size; -~kmeans.dump; - -// Retrieve labels of clustered points +// k = 2 ( -~assignments = Array.new(128); -fork{ - 128.do{ |i| - ~clusters.getLabel(i,{|clusterID| - (i.asString+clusterID).postln; - ~assignments.add(clusterID) - }); - s.sync; - } -} +~kmeans.numClusters_(2).fitPredict(~ds,~ls,{ + ~ls.dump({ + arg dict; + ~fp.categories_(dict); + }); +}); ) -//or faster by sorting the IDs -~clusters.dump{|x|~assignments = x.at("data").atAll(x.at("data").keys.asArray.sort{|a,b|a.asInteger < b.asInteger}).flatten.postln;} - -//Visualise: we're hoping to see colours neatly mapped to quandrants... +// k = 9 ( -d = ((~points + 1) * 0.5).flatten(1).unlace; -w = Window("scatter", Rect(128, 64, 200, 200)); -~colours = [Color.blue,Color.red,Color.green,Color.magenta]; -w.drawFunc = { - Pen.use { - d[0].size.do{|i| - var x = (d[0][i]*200); - var y = (d[1][i]*200); - var r = Rect(x,y,5,5); - Pen.fillColor = ~colours[~assignments[i].asInteger]; - Pen.fillOval(r); - } - } -}; -w.refresh; -w.front; +~kmeans.numClusters_(9).fitPredict(~ds,~ls,{ + ~ls.dump({ + arg dict; + ~fp.categories_(dict); + }); +}); ) -// single point transform on arbitrary value -~inbuf = Buffer.loadCollection(s,0.5.dup); -~kmeans.predictPoint(~inbuf,{|x|x.postln;}); :: - -subsection:: Accessing the means - -We can get and set the means for each cluster, their centroid. - +strong::Incremental Training:: code:: -// with the dataset and kmeans generated and trained in the code above -~centroids = FluidDataSet(s); -~kmeans.getMeans(~centroids, {~centroids.print}); - -// We can also set them to arbitrary values to seed the process -~centroids.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\0, [0.5,0.5], \1, [-0.5,0.5], \2, [0.5,-0.5], \3, [-0.5,-0.5]])])); -~centroids.print -~kmeans.setMeans(~centroids, {~kmeans.predict(~dataSet,~clusters,{~clusters.dump{|x|var count = 0.dup(4); x["data"].keysValuesDo{|k,v|count[v[0].asInteger] = count[v[0].asInteger] + 1;};count.postln}})}); - -// We can further fit from the seeded means -~kmeans.fit(~dataSet) -// then retreive the improved means -~kmeans.getMeans(~centroids, {~centroids.print}); -//subtle in this case but still.. each quadrant is where we seeded it. -:: -subsection:: Cluster-distance Space +// Keep the window open and run this to see where it's at after each iteration -We can get the euclidian distance of a given point to each cluster. This is often referred to as the cluster-distance space as it creates new dimensions for each given point, one distance per cluster. +( +fork{ + ~kmeans.clear; + ~kmeans.numClusters_(9); + ~kmeans.maxIter_(1); + 10.do{ + arg i; + ~kmeans.fitPredict(~ds,~ls,{ + ~ls.dump({ + arg dict; + ~fp.categories_(dict); + }); + }); + "Iteration: %".format(i+1).postln; + 1.wait; + } +} +) -code:: -// with the dataset and kmeans generated and trained in the code above -b = Buffer.sendCollection(s,[0.5,0.5]) -c = Buffer(s) - -// get the distance of our given point (b) to each cluster, thus giving us 4 dimensions in our cluster-distance space -~kmeans.transformPoint(b,c,{|x|x.query;x.getn(0,x.numFrames,{|y|y.postln})}) - -// we can also transform a full dataset -~srcDS = FluidDataSet(s) -~cdspace = FluidDataSet(s) -// make a new dataset with 4 points -~srcDS.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\pp, [0.5,0.5], \np, [-0.5,0.5], \pn, [0.5,-0.5], \nn, [-0.5,-0.5]])])); -~kmeans.transform(~srcDS, ~cdspace, {~cdspace.print}) :: +strong::Seeding the Means:: +code:: -subsection:: Queries in a Synth +( +// peek at a didactic dataset +~ds = FluidDataSet(s).read(FluidFilesPath("../Data/gaussian4.json")); +~ds.dump({ + arg dict; + defer{ + ~fp = FluidPlotter(dict:dict); + }; +}); +) -This is the equivalent of predictPoint, but wholly on the server +// keep that window open and let's display where we will +// seeds the means +( +~seeds = [[0.1,0.1],[0.2,0.2],[0.3,0.3],[0.4,0.4]]; +~seeds.do{ + arg arr, i; + ~fp.addPoint_("seed-%".format(i),arr[0],arr[1],Color.gray,3); +} +) +// do 10 iterations and see where the seeds move to from their starting locations +( +fork{ + // put the seed means that we're going to use in a data set + // so that we can set them with FluidKMeans' method "setMeans" + ~ds_means = FluidDataSet(s).load( + Dictionary.newFrom([ + "cols",2, + "data",Dictionary.newFrom([ + "seed-0",~seeds[0], + "seed-1",~seeds[1], + "seed-2",~seeds[2], + "seed-3",~seeds[3], + ]) + ]) + ); + ~kmeans = FluidKMeans(s); + ~kmeans.setMeans(~ds_means); // set the staring means + // set max iter to 1 so we can seen the progress unfold one step at a time + ~kmeans.maxIter_(1); + // labelset for FluidKMeans to write the categories into + ~ls = FluidLabelSet(s); + + // just do 10 iterations (probably will be enough) + 10.do{ + arg i; + // do one iteration of fit and write the current category lables + ~kmeans.fitPredict(~ds,~ls,{ + ~ls.dump({ + arg dict; + + // set those categories to fluid plotter so we can see the colors + ~fp.categories_(dict); + }); + + // also get the means' current positions and... + ~kmeans.getMeans(~ds_means,{ + ~ds_means.dump({ + arg dict; + + // update their position on the screen + dict["data"].keysValuesDo{ + arg k, v; + ~fp.setPoint_("seed-%".format(k),v[0],v[1],Color.gray,3); + }; + }); + }); + }); + "Iteration: %".format(i+1).postln; + 1.wait; + } +} +) +:: +strong::Distances:: +Accessing the distances to each point's mean code:: + ( -{ - var trig = Impulse.kr(5); - var point = WhiteNoise.kr(1.dup); - var inputPoint = LocalBuf(2); - var outputPoint = LocalBuf(1); - Poll.kr(trig, point, [\pointX,\pointY]); - point.collect{ |p,i| BufWr.kr([p],inputPoint,i)}; - ~kmeans.kr(trig,inputPoint,outputPoint); - Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster); -}.play; +// peek at a didactic dataset +~ds = FluidDataSet(s).read(FluidFilesPath("../Data/gaussian4.json")); +~ds.dump({ + arg dict; + defer{ + ~fp = FluidPlotter(dict:dict).pointSizeScale_(2); + }; +}); ) -// to sonify the output, here are random values alternating quadrant, generated more quickly as the cursor moves rightwards +// keep that window open and run KMeans analysis, then color the points according to their distance from their mean ( -{ - var trig = Impulse.kr(MouseX.kr(0,1).exprange(0.5,ControlRate.ir / 2)); - var step = Stepper.kr(trig,max:3); - var point = TRand.kr(-0.1, [0.1, 0.1], trig) + [step.mod(2).linlin(0,1,-0.6,0.6),step.div(2).linlin(0,1,-0.6,0.6)] ; - var inputPoint = LocalBuf(2); - var outputPoint = LocalBuf(1); - point.collect{|p,i| BufWr.kr([p],inputPoint,i)}; - ~kmeans.kr(trig,inputPoint,outputPoint); - SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps,mul: 0.1); -}.play; +var ds_dist = FluidDataSet(s); +var ds_means = FluidDataSet(s); +var kmeans = FluidKMeans(s).maxIter_(300).fitTransform(~ds,ds_dist); +kmeans.getMeans(ds_means); +~ds.dump({ + arg dict; + ds_dist.dump({ + arg dist_dict; + ds_means.dump({ + arg means_dict; + + dist_dict["data"].keysValuesDo{ + arg id, dist; + + // it provides a distance to every mean, so we'll use the + // smallest value + dist = dist.minItem; + id.postln; + dist.postln; + ~fp.pointColor_(id,Color.hsv(dist * 10,1,1)); + }; + + means_dict["data"].keysValuesDo{ + arg k, v; + ~fp.addPoint_("mean-%".format(k),v[0],v[1],Color.gray,3); + }; + }); + }); +}); ) -:: +:: \ No newline at end of file diff --git a/example-code/sc/LabelSet.scd b/example-code/sc/LabelSet.scd index b3d6b7dd..87402d80 100644 --- a/example-code/sc/LabelSet.scd +++ b/example-code/sc/LabelSet.scd @@ -1,9 +1,140 @@ - code:: + +( ~ls = FluidLabelSet.new(s); +~ls.addLabel("perth","hot"); +~ls.addLabel("huddersfield","cold"); +~ls.print; +) + +( +// set label will add if it doesn't exist +~ls.setLabel("chicago","cold"); +// or update if it does +~ls.setLabel("huddersfield","hot"); +~ls.print; +) + +( +// update label will update an existing label +~ls.updateLabel("perth","dry-heat"); +~ls.updateLabel("huddersfield","wet-winter"); +~ls.updateLabel("chicago","polar-vortex"); +~ls.print; +) -["one", "two", "three"].collect{|x,i| ~ls.addLabel(i, x);}; +// delete label will, ahem, delete a label +( +~ls.deleteLabel("huddersfield"); ~ls.print; +) + +( +// dump gives a dictionary +~ls.dump({ + arg dict; + dict.keysValuesDo{ + arg k, v; + "%:\t%".format(k,v).postln; + }; +}); +) +:: +strong::Labeling Audio Analyses:: +code:: + +( +~src = Buffer.read(s,FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav")); +~ds = FluidDataSet(s); +~ls = FluidLabelSet(s); +) + +// find the average mfcc values across this sound file and add to the dataset +// at the same time, add a label to the label set with the same identifier +( +var id = "ASWINE"; +var mfccs = Buffer(s); +var stats = Buffer(s); +var flat = Buffer(s); +~ls.addLabel(id,"noisy"); +FluidBufMFCC.processBlocking(s,~src,features:mfccs,startCoeff:1); +FluidBufStats.processBlocking(s,mfccs,stats:stats,select:[\mean]); +FluidBufFlatten.processBlocking(s,stats,destination:flat); +~ds.addPoint(id,flat); +~ds.print; +~ls.print; +) + +:: +strong::Other Messages:: +code:: + +// do some analyses and get them all to a label set and data set +( +~srcA = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); +~srcB = Buffer.readChannel(s,FluidFilesPath("Tremblay-CF-ChurchBells.wav"),channels:[0]); +) + +( +fork{ + var labels = ["synth","bells"]; + var mfccs = Buffer(s); + var flat = Buffer(s); + var counter = 0; + + ~ds = FluidDataSet(s); + ~ls = FluidLabelSet(s); + + [~srcA,~srcB].do{ + arg buf, i; + FluidBufMFCC.processBlocking(s,buf,features:mfccs,startCoeff:1); + s.sync; + mfccs.numFrames.do{ + arg frame; + var id = "mfcc-analysis-%".format(counter.asInteger); + FluidBufFlatten.processBlocking(s,mfccs,frame,1,destination:flat); + ~ds.addPoint(id,flat); + ~ls.addLabel(id,labels[i]); + counter = counter + 1; + }; + }; + + ~ds.print; + ~ls.print; +} +) -~ls.free; +// save to disk +~ls.write(Platform.defaultTempDir+/+"mfcc-analysis-labels.json"); + +// read from disk +FluidLabelSet(s).read(Platform.defaultTempDir+/+"mfcc-analysis-labels.json").print; + +// how many lables are there +~ls.size + +// clear +( +~ls.clear; +~ls.print; +) :: +link::Classes/Dictionary:: strong::interface:: +code:: + +( +FluidLabelSet(s).load( + Dictionary.newFrom([ + "cols",1, + "data",Dictionary.newFrom([ + "violin","string", + "viola","string", + "trumpet","brass", + "trombone","brass", + "clarinet","wind" + ]) + ]) +).print; +) + +:: \ No newline at end of file diff --git a/example-code/sc/Loudness.scd b/example-code/sc/Loudness.scd index 7df5f418..5a1eb6a5 100644 --- a/example-code/sc/Loudness.scd +++ b/example-code/sc/Loudness.scd @@ -1,77 +1,65 @@ - - code:: -//create a monitoring bus for the descriptors -b = Bus.new(\control,0,2); - -//create a monitoring window for the values -( -w = Window("Loudness Monitor", Rect(10, 10, 220, 65)).front; -c = Array.fill(2, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)}); -c[0].string = ("Loudness: "); -c[1].string = ("Peak: "); +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -a = Array.fill(2, {arg i; - StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center); -}); -) - -//routine to update the parameters ( -r = Routine { - { - b.get({ arg val; - { - if(w.isClosed.not) { - val.do({arg item,index; - a[index].string = item.round(0.01)}) - } - }.defer - }); - 0.1.wait; - }.loop -}.play -) +~synth = { + var src = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var loudness = FluidLoudness.kr(src); + var noise = PinkNoise.ar(loudness[0].dbamp); -//basic test, with default values -( -x = {var source = PinkNoise.ar(0.25); - Out.kr(b, FluidLoudness.kr(source)); - source.dup; + loudness[0].poll(label:"Loudness:"); + loudness[1].poll(label:"TruePeak:"); + 0.poll(label:"---------------------"); + + src = DelayN.ar(src,0.1,1024*SampleDur.ir); // delay to compensate for latency + [src * -6.dbamp,noise]; }.play; ) -//free this -x.free +:: +strong::EBU Standards:: +The EBU standard specifies that the window should be 400ms long, and update every 100ms, for instantaneous loudness. Here we set the windowSize and hopSize appropriately using SampleRate.ir. Various test signals are loaded. +code:: -//the EBU standard specifies that the window should be 400ms long, and update every 100ms, for instantaneous loudness. At SR=44100, this means the following settings. Various test signals are loaded. ( -x = { +~synth = { arg freq=220, type = 1, noise = 0; - var source = PinkNoise.ar(noise) + Select.ar(type,[DC.ar(),SinOsc.ar(freq,mul:0.1), VarSaw.ar(freq,mul:0.1), Saw.ar(freq,0.1), Pulse.ar(freq,mul:0.1)]); - Out.kr(b, FluidLoudness.kr(source,windowSize:17640,hopSize:4410,maxWindowSize:17640)); + var source = PinkNoise.ar(noise) + Select.ar(type,[ + DC.ar(), + SinOsc.ar(freq,mul:0.1), + VarSaw.ar(freq,mul:0.1), + Saw.ar(freq,0.1), + Pulse.ar(freq,mul:0.1) + ]); + var loudness = FluidLoudness.kr(source,windowSize:SampleRate.ir*0.4,hopSize:SampleRate.ir*0.1); + + loudness[0].poll(label:"Loudness:"); + loudness[1].poll(label:"TruePeak:"); + 0.poll(label:"---------------------"); + source.dup; }.play; ) + // change the various frequencies to see the impact of the filter for the loudness. The TruePeak is steady. -x.set(\freq, 440) -x.set(\freq, 110) -x.set(\freq, 55) -x.set(\freq, 3000) -x.set(\freq, 9000) +~synth.set(\freq, 440) +~synth.set(\freq, 110) +~synth.set(\freq, 55) +~synth.set(\freq, 3000) +~synth.set(\freq, 9000) // adding harmonics, by changing to triangle (2), saw (3) or square (4) shows that spectral algo are more resilient when signal are richer -x.set(\type, 2) -x.set(\type, 3) -x.set(\type, 4) +~synth.set(\type, 2) +~synth.set(\type, 3) +~synth.set(\type, 4) // adding noise shows its impact on loudness -x.set(\noise, 0.25) +~synth.set(\noise, 0.25) // and removing the oscilator -x.set(\type, 0) +~synth.set(\type, 0) // and measuring silence -x.set(\noise, 0) +~synth.set(\noise, 0) :: diff --git a/example-code/sc/Normalize.scd b/example-code/sc/Normalize.scd index 916cfb7a..cf4d1eed 100644 --- a/example-code/sc/Normalize.scd +++ b/example-code/sc/Normalize.scd @@ -1,91 +1,139 @@ code:: -s.boot; -//Preliminaries: we want some audio, a couple of FluidDataSets, some Buffers and a FluidNormalize -// FluidNormalize.dumpAllMethods + +~src = Buffer.read(s,FluidFilesPath("Tremblay-UW-ComplexDescent-M.wav")); + +// pitch analysis ( -~audiofile = FluidFilesPath("Tremblay-ASWINE-ScratchySynth-M.wav"); -~raw = FluidDataSet(s); -~norm = FluidDataSet(s); -~pitch_feature = Buffer.new(s); -~stats = Buffer.alloc(s, 7, 2); -~normalizer = FluidNormalize(s); +fork({ + ~pitch = Buffer(s); + FluidBufPitch.processBlocking(s,~src,features:~pitch); + s.sync; + + // using plot here rather than FluidWaveform to see the scales (on the left axis) + ~pitch.plot(separately:true); +},AppClock); ) -// Load audio and run a pitch analysis, which gives us pitch and pitch confidence (so a 2D datum) +// get analyses from buffer to dataset ( -~audio = Buffer.read(s,~audiofile); -FluidBufPitch.process(s,~audio, features: ~pitch_feature); +~ds = FluidDataSet(s).fromBuffer(~pitch); +~ds.print; ) -// Divide the time series in to 10, and take the mean of each segment and add this as a point to -// the 'raw' FluidDataSet +// normalize it ( -{ - var trig = LocalIn.kr(1, 1); - var buf = LocalBuf(2, 1); - var count = PulseCount.kr(trig) - 1; - var chunkLen = (~pitch_feature.numFrames / 10).asInteger; - var stats = FluidBufStats.kr( - source: ~pitch_feature, startFrame: count * chunkLen, - numFrames: chunkLen, stats: ~stats, trig: (trig * (count <=9)), blocking:1 - ); - var rd = BufRd.kr(2, ~stats, DC.kr(0), 0, 1);// pick only mean pitch and confidence - var wr1 = BufWr.kr(rd[0], buf, DC.kr(0)); - var wr2 = BufWr.kr(rd[1], buf, DC.kr(1)); - var dsWr = FluidDataSetWr.kr(~raw, buf: buf, idNumber: count, trig: Done.kr(stats)); - LocalOut.kr( Done.kr(dsWr)); - Poll.kr(trig,count,\count); - FreeSelf.kr(count - 9); -}.play; +~normalizer = FluidNormalize(s).fitTransform(~ds,~ds); +~ds.print; ) -// Normalize and load to language-side array +// get it back to a buffer, plot the buffer to see the ranges ( -~rawarray = Array.new(10); -~normedarray= Array.new(10); -~normalizer.fitTransform(~raw,~norm, { - ~raw.dump{|x| 10.do{|i| - ~rawarray.add(x["data"][i.asString]) - }}; - ~norm.dump{|x| 10.do{|i| - ~normedarray.add(x["data"][i.asString]) - }}; -}); +fork({ + ~normed = Buffer(s); + ~ds.toBuffer(~normed); + s.sync; + ~normed.plot(separately:true); +},AppClock); ) -//Plot side by side. Before normalization the two dimensions have radically different scales -//which can be unhelpful in many cases +:: +strong::Server side queries:: +code:: +( +{ + var src = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var pitch = FluidPitch.kr(src); + var inputPoint = LocalBuf(2); + var outputPoint = LocalBuf(2); + var trig = Impulse.kr(30); + var normed; + FluidKrToBuf.kr(pitch,inputPoint); + ~normalizer.kr(trig,inputPoint,outputPoint); + normed = FluidBufToKr.kr(outputPoint); + + pitch[0].poll(label:"pitch "); + normed[0].poll(label:"normalized pitch "); + 0.poll(label:"-------------------"); + pitch[1].poll(label:"pitch confidence "); + normed[1].poll(label:"normalized pitch confidence "); + 0.poll(label:"--------------------------------------"); + 0.poll(label:"--------------------------------------"); + src.dup; +}.play; +) +:: +strong::Comparing Scalers:: +code:: + +~src = Buffer.readChannel(s,FluidFilesPath("Olencki-TenTromboneLongTones-M.wav"),channels:[0]); + +// spectral analysis ( -(~rawarray ++ 0).flop.plot("Unnormalized",Rect(0,0,400,400),minval:0,maxval:[5000,1]).plotMode=\bars; -(~normedarray ++ 0).flop.plot("Normalized",Rect(410,0,400,400)).plotMode=\bars; +~select = [\skewness,\flatness]; +~features = Buffer(s); +FluidBufSpectralShape.processBlocking(s,~src,features:~features,select:~select); +~ds = FluidDataSet(s).fromBuffer(~features); +~ds.print; ) -// single point transform on arbitrary value -~inbuf = Buffer.loadCollection(s,0.5.dup); -~outbuf = Buffer.new(s); -~normalizer.transformPoint(~inbuf,~outbuf,{|x|x.postln;x.getn(0,2,{|y|y.postln;};)}); +// scale using Normalize, Standardize, and RobustScale +( +~ds_norm = FluidDataSet(s); +~ds_stan = FluidDataSet(s); +~ds_robu = FluidDataSet(s); +FluidNormalize(s).fitTransform(~ds,~ds_norm); +FluidStandardize(s).fitTransform(~ds,~ds_stan); +~robu = FluidRobustScale(s).fitTransform(~ds,~ds_robu); +~ds_norm.print; +~ds_stan.print; +~ds_robu.print; +) -//Server side queries +// plot the three scalers ( -{ - var audio = BufRd.ar(1,~audio,LFSaw.ar(BufDur.ir(~audio).reciprocal).range(0, BufFrames.ir(~audio))); - var counter = Stepper.ar(Impulse.ar(ControlRate.ir),max:99); - var trig = A2K.kr(HPZ1.ar(counter) < 0); - //average 100 frames: one could use the MovingAverage extension here - var avg; - var inputPoint = LocalBuf(2); - var outputPoint = LocalBuf(2); - var avgBuf = LocalBuf(100,2); - //running average of pitch features - BufWr.kr(FluidPitch.kr(audio),avgBuf,phase:counter); - avg = Mix.new(BufRd.kr(2, avgBuf, phase:100.collect{|x|x})) * 0.01; - //assemble data point - BufWr.kr(avg[0],inputPoint,0); - BufWr.kr(avg[1],inputPoint,1); - ~normalizer.kr(trig,inputPoint,outputPoint); - Poll.kr(trig,BufRd.kr(1,inputPoint,[0,1]),["pitch (raw)", "confidence (raw)"]); - Poll.kr(trig,BufRd.kr(1,outputPoint,[0,1]),["pitch (normalized)", "confidence (normalized)"]) -}.play; +~ds_norm.dump({ + arg dict_norm; + ~ds_stan.dump({ + arg dict_stan; + ~ds_robu.dump({ + arg dict_robu; + + defer{ + var win = Window("Comparing Scalers",Rect(0,0,1500,500)); + var comp = CompositeView(win,win.bounds); + comp.layout_( + HLayout( + *[dict_norm,dict_stan,dict_robu].collect{ + arg dict; + FluidPlotter( + win, + standalone:false, + dict:dict, + xmin:-3, + xmax:3, + ymin:-3, + ymax:3 + ); + }; + ) + ); + + UserView(win,win.bounds) + .drawFunc_{ + Pen.stringInRect("All plots are the same data with different scalers.\nAll axes are from -3 to 3.",Rect(20,30,win.bounds.width * 0.3,40),color:Color.blue); + ["Normalize","Standardize","RobustScale"].do{ + arg name, i; + Pen.stringAtPoint(name,Point((i * win.bounds.width * 0.33) + 20,15),color:Color.red); + }; + Pen.stringInRect("In this case, RobustScale keeps more of the data centred around 0 in both\ndimensions than Standardize (and of course better than Normalize).\nNotice however that there's a lot of data \"off-plot\" with RobustScale.",Rect(win.bounds.width * 0.67,win.bounds.height * 0.88,win.bounds.width * 0.3,win.bounds.height * 0.3),color:Color.blue); + }; + + win.front; + }; + }); + }); +}); ) -:: + +:: \ No newline at end of file diff --git a/example-code/sc/NoveltySlice.scd b/example-code/sc/NoveltySlice.scd index 305d3177..874aedd3 100644 --- a/example-code/sc/NoveltySlice.scd +++ b/example-code/sc/NoveltySlice.scd @@ -1,23 +1,35 @@ - code:: -//load some sounds -b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); - -// basic param (the process add a latency of windowSize samples -{var sig = PlayBuf.ar(1,b,loop:1); [FluidNoveltySlice.ar(sig,0,11,0.33) * 0.5, DelayN.ar(sig, 1, (512 * (((11 + 1) / 2).asInteger + ((1 + 1) / 2).asInteger + 1)) / s.sampleRate, 0.2)]}.play -// other parameters -{var sig = PlayBuf.ar(1,b,loop:1); [FluidNoveltySlice.ar(sig, 1, 31, 0.009, 4, 100, 128, 32) * 0.5, DelayN.ar(sig, 1, (32 * (((31 + 1)/2).asInteger + ((4 + 1) / 2).asInteger + 1))/ s.sampleRate,0.2)]}.play +~src = Buffer.read(s,FluidFilesPath("Tremblay-FMTriDist-M.wav")); -// More musical, novelty-trigged autopan ( { - var sig, trig, syncd, pan; - sig = PlayBuf.ar(1,b,loop:1); - trig = FluidNoveltySlice.ar(sig, 0, 11, 0.25, 5, 1, 128, 32); - syncd = DelayN.ar(sig, 1, (32 * (((11 + 1)/2).asInteger + ((5 + 1) / 2).asInteger + 1))/ s.sampleRate); - pan = TRand.ar(-1,1,trig); - Pan2.ar(syncd,pan); -}.play + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var slicer = FluidNoveltySlice.ar(sig,threshold:0.61); + sig = DelayN.ar(sig,0.1,1024*SampleDur.ir); // compensate for latency + [sig,slicer]; +}.play; ) + :: +strong::Tweaking Parameters:: +code:: + +~src = Buffer.read(s,FluidFilesPath("Constanzo-PreparedSnare-M.wav")); + +( +~synth = { + arg kernelSize = 3, threshold = 0.8, minSliceLength = 2, windowSize = 1024; + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var slicer = FluidNoveltySlice.ar(sig,0,kernelSize,threshold,1,minSliceLength,windowSize,maxKernelSize:41); + sig = DelayN.ar(sig,0.1,windowSize*SampleDur.ir); // compensate for latency + [sig,slicer]; +}.play; +) + +// try some different params +~synth.set(\kernelSize,5,\threshold,0.61,\minSliceLength,3); +~synth.set(\kernelSize,41,\threshold,0.35,\minSliceLength,1); +~synth.set(\kernelSize,3,\threshold,0.5,\minSliceLength,1); + +:: \ No newline at end of file diff --git a/example-code/sc/OnsetSlice.scd b/example-code/sc/OnsetSlice.scd index 2574a043..7c949138 100644 --- a/example-code/sc/OnsetSlice.scd +++ b/example-code/sc/OnsetSlice.scd @@ -1,23 +1,36 @@ code:: -//load some sounds -b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -// basic param (the process add a latency of windowSize samples -{var sig = PlayBuf.ar(1,b,loop:1); [FluidOnsetSlice.ar(sig) * 0.5, DelayN.ar(sig, 1, 1024/ s.sampleRate)]}.play +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); + +( +{ + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var onsets = FluidOnsetSlice.ar(sig); + sig = DelayN.ar(sig, 1, 1024 * SampleDur.ir); // compensate for a latency of windowSize samples + [sig,onsets]; +}.play +) // other parameters -{var sig = PlayBuf.ar(1,b,loop:1); [FluidOnsetSlice.ar(sig, 9, 0.15, 45, 9, 0, 128, 64) * 0.5, DelayN.ar(sig, 1, (128)/ s.sampleRate)]}.play +( +{ + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var onsets = FluidOnsetSlice.ar(sig,metric:9,threshold:0.15,minSliceLength:45,filterSize:9,frameDelta:0,windowSize:128,hopSize:64); + sig = DelayN.ar(sig, 1, 128 * SampleDur.ir); + [sig, onsets] +}.play +) // More musical, onset-trigged autopan ( { - var sig, trig, syncd, pan; - sig = PlayBuf.ar(1,b,loop:1); - trig = FluidOnsetSlice.ar(sig, 7, 0.2, 100, 9, 0, 128); - syncd = DelayN.ar(sig, 1, (128 / s.sampleRate)); //one could add a bit more delay here to make sure to catch the attack + var sig, trig, pan; + sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + trig = FluidOnsetSlice.ar(sig,metric:7,threshold:0.2,minSliceLength:100,filterSize:9,frameDelta:0,windowSize:128); + sig = DelayN.ar(sig, 1, 128 * SampleDur.ir); //one could add a bit more delay here to make sure to catch the attack pan = Stepper.ar(trig,-1,-1,1,1); - Pan2.ar(syncd,pan); + Pan2.ar(sig,pan); }.play ) :: diff --git a/example-code/sc/SKMeans.scd b/example-code/sc/SKMeans.scd index 777dd084..1cb19322 100644 --- a/example-code/sc/SKMeans.scd +++ b/example-code/sc/SKMeans.scd @@ -1,33 +1,37 @@ code:: -( //Make some clumped 2D points and place into a DataSet -~points = (4.collect{ - 64.collect{(1.sum3rand) + [1,-1].choose}.clump(2) - }).flatten(1) * 0.5; -fork{ - ~dataSet = FluidDataSet(s); - d = Dictionary.with( - *[\cols -> 2,\data -> Dictionary.newFrom( - ~points.collect{|x, i| [i, x]}.flatten)]); - s.sync; - ~dataSet.load(d, {~dataSet.print}); -} +( +var points = 4.collect{ + 64.collect{(1.sum3rand) + [1,-1].choose}.clump(2) +}.flatten(1) * 0.5; + +var dict = Dictionary.with( + *[ + \cols -> 2, + \data -> Dictionary.newFrom( + points.collect{|x, i| [i, x]}.flatten) + ] +); + +~dataSet = FluidDataSet(s).load(dict).print; ) // Create an SKMeans instance and a LabelSet for the cluster labels in the server +( ~clusters = FluidLabelSet(s); ~skmeans = FluidSKMeans(s); +) // Fit into 4 clusters ( ~skmeans.fitPredict(~dataSet,~clusters,action: {|c| - "Fitted.\n # Points in each cluster:".postln; - c.do{|x,i| - ("Cluster" + i + "->" + x.asInteger + "points").postln; - } - }); + "Fitted.\n # Points in each cluster:".postln; + c.do{|x,i| + ("Cluster" + i + "->" + x.asInteger + "points").postln; + } +}); ) // Cols of SKMeans should match DataSet, size is the number of clusters @@ -36,32 +40,60 @@ fork{ ~skmeans.size; ~skmeans.dump; -// Retrieve labels of clustered points by sorting the IDs -~clusters.dump{|x|~assignments = x.at("data").atAll(x.at("data").keys.asArray.sort{|a,b|a.asInteger < b.asInteger}).flatten.postln;} - -//Visualise: we're hoping to see colours neatly mapped to quandrants... ( -d = ((~points + 1) * 0.5).flatten(1).unlace; -w = Window("scatter", Rect(128, 64, 200, 200)); -~colours = [Color.blue,Color.red,Color.green,Color.magenta]; -w.drawFunc = { - Pen.use { - d[0].size.do{|i| - var x = (d[0][i]*200); - var y = (d[1][i]*200); - var r = Rect(x,y,5,5); - Pen.fillColor = ~colours[~assignments[i].asInteger]; - Pen.fillOval(r); - } - } +var norm_ds = FluidDataSet(s); +FluidNormalize(s).fitTransform(~dataSet,norm_ds); // normalize for ease of viewing +norm_ds.dump{ + arg data; + ~clusters.dump{ + arg labels; + defer{ + FluidPlotter(dict:data).categories_(labels); + }; + norm_ds.free + }; }; -w.refresh; -w.front; ) // single point query on arbitrary value -~inbuf = Buffer.loadCollection(s,0.5.dup); -~skmeans.predictPoint(~inbuf,{|x|x.postln;}); +( +~inbuf = Buffer.loadCollection(s,{rrand(-1.0,1.0)} ! 2); +~skmeans.predictPoint(~inbuf,{|x|"cluster: ".post; x.postln;}); +) + +:: +strong::Incremental Training:: +code:: + +( +// peek at a the dataset +~dataSet.dump({ + arg dict; + defer{ + ~fp = FluidPlotter(dict:dict,xmin:-1,ymin:-1).pointSizeScale_(3); + }; +}); +) + +// now keeping that window open, do 10 fittings, watching how it changes after each +( +fork{ + ~kmeans = FluidSKMeans(s,4,maxIter:1); + ~ls = FluidLabelSet(s); + 10.do{ + arg i; + ~kmeans.fitPredict(~dataSet,~ls,{ + ~ls.dump({ + arg dict; + ~fp.categories_(dict); + }); + }); + "Iteration: %".format(i+1).postln; + 1.wait; + } +} +) + :: subsection:: Accessing the means @@ -69,14 +101,39 @@ subsection:: Accessing the means We can get and set the means for each cluster, their centroid. code:: + // with the dataset and skmeans generated and trained in the code above +( ~centroids = FluidDataSet(s); ~skmeans.getMeans(~centroids, {~centroids.print}); +) // We can also set them to arbitrary values to seed the process -~centroids.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\0, [0.5,0.5], \1, [-0.5,0.5], \2, [0.5,-0.5], \3, [-0.5,-0.5]])])); -~centroids.print -~skmeans.setMeans(~centroids, {~skmeans.predict(~dataSet,~clusters,{~clusters.dump{|x|var count = 0.dup(4); x["data"].keysValuesDo{|k,v|count[v[0].asInteger] = count[v[0].asInteger] + 1;};count.postln}})}); +( +~centroids.load( + Dictionary.newFrom([ + \cols, 2, + \data, Dictionary.newFrom([ + \0, [0.5,0.5], + \1, [-0.5,0.5], + \2, [0.5,-0.5], + \3, [-0.5,-0.5] + ]) + ]) +); +~skmeans.setMeans(~centroids); +~skmeans.predict(~dataSet,~clusters); +~clusters.dump{ + arg dict; + var count = 0.dup(4); + dict["data"].keysValuesDo{ + arg k, v; + count[v[0].asInteger] = count[v[0].asInteger] + 1; + }; + "number of points in each cluster:".postln; + count.postln +}; +) // We can further fit from the seeded means ~skmeans.fit(~dataSet) @@ -91,18 +148,38 @@ We can get the spherical distance of a given point to each cluster. SKMeans diff code:: // with the dataset and skmeans generated and trained in the code above -b = Buffer.sendCollection(s,[0.5,0.5]) -c = Buffer(s) +( +~inbuf = Buffer.sendCollection(s,[0.5,0.5]); +~encoded = Buffer(s) +) // get the distance of our given point (b) to each cluster, thus giving us 4 dimensions in our cluster-distance space -~skmeans.encodePoint(b,c,{|x|x.query;x.getn(0,x.numFrames,{|y|y.postln})}) +( +~skmeans.encodePoint(~inbuf,~encoded); +~encoded.getn(0,4,action:{ + arg dists; + dists.postln; +}); +) // we can also encode a full dataset -~srcDS = FluidDataSet(s) -~cdspace = FluidDataSet(s) +( +~cdspace = FluidDataSet(s); // make a new dataset with 4 points -~srcDS.load(Dictionary.newFrom([\cols, 2, \data, Dictionary.newFrom([\pp, [0.5,0.5], \np, [-0.5,0.5], \pn, [0.5,-0.5], \nn, [-0.5,-0.5]])])); -~skmeans.encode(~srcDS, ~cdspace, {~cdspace.print}) +~srcDS = FluidDataSet(s).load( + Dictionary.newFrom([ + \cols, 2, + \data, Dictionary.newFrom([ + \pp, [0.5,0.5], + \np, [-0.5,0.5], + \pn, [0.5,-0.5], + \nn, [-0.5,-0.5] + ]) + ]) +); +~skmeans.encode(~srcDS, ~cdspace); +~cdspace.print; +) :: subsection:: Queries in a Synth @@ -112,14 +189,14 @@ This is the equivalent of predictPoint, but wholly on the server code:: ( { - var trig = Impulse.kr(5); - var point = WhiteNoise.kr(1.dup); - var inputPoint = LocalBuf(2); - var outputPoint = LocalBuf(1); - Poll.kr(trig, point, [\pointX,\pointY]); - point.collect{ |p,i| BufWr.kr([p],inputPoint,i)}; - ~skmeans.kr(trig,inputPoint,outputPoint); - Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster); + var trig = Impulse.kr(5); + var point = WhiteNoise.kr(1.dup); + var inputPoint = LocalBuf(2); + var outputPoint = LocalBuf(1); + Poll.kr(trig, point, [\pointX,\pointY]); + point.collect{ |p,i| BufWr.kr([p],inputPoint,i)}; + ~skmeans.kr(trig,inputPoint,outputPoint); + Poll.kr(trig,BufRd.kr(1,outputPoint,0,interpolation:0),\cluster); }.play; ) @@ -129,11 +206,11 @@ code:: var trig = Impulse.kr(MouseX.kr(0,1).exprange(0.5,ControlRate.ir / 2)); var step = Stepper.kr(trig,max:3); var point = TRand.kr(-0.1, [0.1, 0.1], trig) + [step.mod(2).linlin(0,1,-0.6,0.6),step.div(2).linlin(0,1,-0.6,0.6)] ; - var inputPoint = LocalBuf(2); - var outputPoint = LocalBuf(1); + var inputPoint = LocalBuf(2); + var outputPoint = LocalBuf(1); point.collect{|p,i| BufWr.kr([p],inputPoint,i)}; - ~skmeans.kr(trig,inputPoint,outputPoint); - SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps,mul: 0.1); + ~skmeans.kr(trig,inputPoint,outputPoint); + SinOsc.ar((BufRd.kr(1,outputPoint,0,interpolation:0) + 69).midicps,mul: 0.1); }.play; ) diff --git a/example-code/sc/Sines.scd b/example-code/sc/Sines.scd index 24f8af08..43735fac 100644 --- a/example-code/sc/Sines.scd +++ b/example-code/sc/Sines.scd @@ -1,26 +1,55 @@ CODE:: -// load some audio to play -b = Buffer.read(s,FluidFilesPath("Tremblay-AaS-SynthTwoVoices-M.wav")); -// run with large parameters - left is sinusoidal model, right is residual -{FluidSines.ar(PlayBuf.ar(1,b,loop:1),detectionThreshold: -40, minTrackLen: 2, windowSize: 2048, fftSize: 8192)}.play +~src = Buffer.readChannel(s,FluidFilesPath("Tremblay-BeatRemember.wav"),channels:[0]); -// interactive parameters with a narrower bandwidth -{FluidSines.ar(PlayBuf.ar(1,b,loop:1), 30, MouseX.kr(-140,-10),MouseY.kr(-110,-10),MouseY.kr(-140,-40), 10 , windowSize: 1000, hopSize: 200, fftSize: 4096)}.play +( +~synth = { + arg which = 0, detectionThreshold = -96, minTrackLen = 15; + var src = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var sines, residual; + # sines, residual = FluidSines.ar(src,detectionThreshold:detectionThreshold,minTrackLen:minTrackLen); + Select.ar(which,[sines,residual]).dup; +}.play; +) -// null test (the process add a latency of (( hopSize * minTrackLen) + windowSize) samples -{var sig = PlayBuf.ar(1,b,loop:1); [FluidSines.ar(sig).sum - DelayN.ar(sig, 1, ((( 512 * 15) + 1024)/ s.sampleRate))]}.play +~synth.set(\which,1) // residual +~synth.set(\which,0) // back to sinusoids -// as the algorithm resynthesize the sinusoidal peaks, we would expect to get it to work almost perfectly on a sine wave, with these settings that tell the process to tolerate everything as a sinusoid, even short and quiet peaks -{FluidSines.ar(SinOsc.ar(mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 200,trackFreqRange: 1000,trackProb: 0)}.play; +// try some different parameters -// we can listen to the artefact in solo, amplifying it by 30dB, to hear the 'lobes' - not bad at all! -{FluidSines.ar(SinOsc.ar(mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 200,trackFreqRange: 1000,trackProb: 0)[1].dup * Line.ar(0,30,1).dbamp}.play; +// tracks can be short but the detection threshold is higher than the default +~synth.set(\detectionThreshold,-40,\minTrackLen,1) -// as this is a windowed process, the frequency of the peak is good for that full window, and therefore interesting artefacts appear when the pitch is changing. -{FluidSines.ar(SinOsc.ar(LFTri.kr(0.1).exprange(220,880),mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 300,trackFreqRange: 1000,trackProb: 0)}.play; +// increase the minimum track length +~synth.set(\detectionThreshold,-40,\minTrackLen,15) -// if we solo and amplify the artefacts, they are much more apparent (and interesting) -{FluidSines.ar(SinOsc.ar(LFTri.kr(0.1).exprange(220,880),mul: 0.1),detectionThreshold: -144,birthLowThreshold: -144,birthHighThreshold: -144,minTrackLen: 1,trackMagRange: 300,trackFreqRange: 1000,trackProb: 0)[1].dup * Line.ar(0,30,1).dbamp}.play; +// lower the threshold but increase the track length drastically +~synth.set(\detectionThreshold,-80,\minTrackLen,50) + +// increase the threshold drastically but lower the minimum track length +~synth.set(\detectionThreshold,-24,\minTrackLen,1) :: +strong::a little more explanation:: +With these settings everything in the sound is considered a sinusoid, even short and quiet peaks. + +Because the decomposition is a windowed process, the detected sinusoidal peaks are located in time based on the window of analysis. When the oscillator changes (even slowly) over time we hear the artefact in the residual output. +code:: + +( +~synth = { + arg which = 0; + var stable = SinOsc.ar(69.midicps,0,0.1); + var oscillating = SinOsc.ar(SinOsc.kr(0.1,0,12,57).midicps,0,0.1); + var sig = SelectX.ar(which.lag(0.1),[stable,oscillating]); + var sines, residual; + # sines, residual = FluidSines.ar(sig,76,-144,-144,-144,1,0,200,1000,0); + [sines, residual * ((which*40) + 1).lag(0.1)] +}.play +) + +~synth.set(\which,1); + +~synth.set(\which,0); + +:: \ No newline at end of file diff --git a/example-code/sc/SpectralShape.scd b/example-code/sc/SpectralShape.scd index 752c9faf..d6a166c6 100644 --- a/example-code/sc/SpectralShape.scd +++ b/example-code/sc/SpectralShape.scd @@ -1,259 +1,55 @@ - - code:: -//create a monitoring bus for the descriptors -b = Bus.new(\control,0,7); - -//create a monitoring window for the values - -( -w = Window("spectral Shape Monitor", Rect(10, 10, 220, 190)).front; - -c = Array.fill(7, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)}); -c[0].string = ("Centroid: "); -c[1].string = ("Spread: "); -c[2].string = ("Skewness: "); -c[3].string = ("Kurtosis: "); -c[4].string = ("Rolloff: "); -c[5].string = ("Flatness: "); -c[6].string = ("Crest: "); - -a = Array.fill(7, {arg i; - StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center); -}); -) - -//run the window updating routine. -( -r = Routine { - { - - b.get({ arg val; - { - if(w.isClosed.not) { - val.do({arg item,index; - a[index].string = item.round(0.01)}) - } - }.defer - }); - - 0.01.wait; - }.loop - -}.play -) - -//play a simple sound to observe the values -( - { - var source; - source = BPF.ar(WhiteNoise.ar(), 330, 55/330); - Out.kr(b,FluidSpectralShape.kr(source)); - source.dup; - }.play; -) -:: - -STRONG::A commented tutorial on how each descriptor behaves with test signals: :: - -CODE:: -// as above, create a monitoring bus for the descriptors -b = Bus.new(\control,0,7); -//again, create a monitoring window for the values -( -w = Window("Spectral Shape Monitor", Rect(10, 10, 220, 190)).front; - -c = Array.fill(7, {arg i; StaticText(w, Rect(10, i * 25 + 10, 135, 20)).background_(Color.grey(0.7)).align_(\right)}); -c[0].string = ("Centroid: "); -c[1].string = ("Spread: "); -c[2].string = ("Skewness: "); -c[3].string = ("Kurtosis: "); -c[4].string = ("Rolloff: "); -c[5].string = ("Flatness: "); -c[6].string = ("Crest: "); - -a = Array.fill(7, {arg i; - StaticText(w, Rect(150, i * 25 + 10, 60, 20)).background_(Color.grey(0.7)).align_(\center); -}); -) - -// this time, update a little more slowly. -( -r = Routine { - { - - b.get({ arg val; - { - if(w.isClosed.not) { - val.do({arg item,index; - a[index].string = item.round(0.01)}) - } - }.defer - }); - - 0.2.wait; - }.loop - -}.play -) +~src = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); -// first, a sine wave ( -x = { - arg freq=220; - var source; - source = SinOsc.ar(freq,mul:0.1); - Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate)); - source.dup; -}.play; -) - -// at 220, the centroid is on the frequency, the spread is narrow, but as wide as the FFT Hann window ripples, the skewness is high as we are low and therefore far left of the middle bin (aka half-Nyquist), the Kurtosis is incredibly high as we have a very peaky spectrum. The rolloff is slightly higher than the frequency, taking into account the FFT windowing ripples, the flatness is incredibly low, as we have one peak and not much else, and the crest is quite high, because most of the energy is in a few peaky bins. - -x.set(\freq, 440) - -// at 440, the skewness has changed (we are nearer the middle of the spectrogram) and the Kurtosis too, although it is still so high it is quite in the same order of magnitude. The rest is stable, as expected. +~synth = { + arg power = 1; + var sig = PlayBuf.ar(1,~src,BufRateScale.ir(~src),loop:1); + var spec = FluidSpectralShape.kr(sig,[\centroid,\spread],unit:1,power:power); + var noise = WhiteNoise.ar(1); + var q = pow(2,spec[1]/6); + q = pow(q,0.5) / (q-1); -x.set(\freq, 11000) + spec[0].poll(label:"centroid midi"); + spec[1].poll(label:"spread midi"); + spec[0].midicps.poll(label:"centroid hz"); + q.poll(label:"q"); + 0.poll(label:"---------------------------"); -// at 11kHz, kurtosis is still in the thousand, but skewness is almost null, as expected. - -x.free - -// second, broadband noise -( -x = { - arg type = 0; - var source; - source = Select.ar(type,[WhiteNoise.ar(0.1),PinkNoise.ar(0.1)]); - Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate)); - source.dup; + noise = BPF.ar(noise,spec[0].midicps.lag(0.1),1); + noise = noise * FluidLoudness.kr(sig,[\loudness]).dbamp * 10.dbamp; + Limiter.ar([sig,noise]); }.play; ) -// white noise has a linear repartition of energy, so we would expect a centroid in the middle bin (aka half-Nyquist) with a spread covering the full range (+/- a quarter-Nyquist), with a skewness almost null since we are centered, and a very low Kurtosis since we are flat. The rolloff should be almost at Nyquist, the flatness as high as it gets, and the crest quite low. - -x.set(\type, 1) - -// pink noise has a drop of 3dB per octave across the spectrum, so we would, by comparison, expect a lower centroid, a slighly higher skewness and kurtosis, a lower rolloff, a slighly lower flatness and a higher crest for the larger low-end energy. +~synth.set(\power,1); -x.free - -// third, bands of noise -( -x = { - arg type = 0; - var source, chain; - chain = FFT(LocalBuf(1024), WhiteNoise.ar(0.5)); - chain = chain.pvcollect(1024, {arg mag,phase;[mag,phase]},5,11,1); - source = Select.ar(type,[ - BPF.ar(BPF.ar(WhiteNoise.ar(0.5),330,0.666),330,0.666), - IFFT(chain)]); - Out.kr(b, VarLag.kr(FluidSpectralShape.kr(source),1024/s.sampleRate)); - source.dup; -}.play; -) - -// a second-order bandpass filter on whitenoise, centred on 330Hz with one octave bandwidth, gives us a centroid quite high. This is due to the exponential behaviour of the filter, with a gentle slope. Observe the spectral analyser: - -s.freqscope - -// at first it seems quite centred, but then flip the argument FrqScl to lin(ear) and observe how high the spectrum goes. If we set it to a brickwall spectral filter tuned on the same frequencies: - -x.set(\type, 1) - -// we have a much narrower register, and our centroid and spread, as well as the kurtosis and flatness, agrees with this reading. - -x.free - -//fourth, equally spaced sines -( -x = { - arg freq = 220; - var source; - source = Mix.fill(7, {arg ind; SinOsc.ar(freq + (ind * (220 / 6)), 0, 0.02)}); - Out.kr(b,FluidSpectralShape.kr(source)); - source.dup; -}.play; -) - -// this example shows a similar result to the brickwall spectral bandpass above. If we move the central frequency nearer the half-Nyquist: - -x.set(\freq, 8800) - -// we can observe that the linear spread is kept the same, since there is the same linear distance in Hz between our frequencies. Skewness is a good indication here of where we are in the spectrum with the shape. :: +strong::logarithmic scale:: +The computation of the spectral centroid can also be done considering a logarithmic pitch scale and the power of the magnitudes. This yields values that are generally considered to be more in line with perception, for instance where the shape is often drawn and described in logarithmic terms, i.e., dB per octave. +Compare the values of the centroid and the spread in both scales. The lower the frequency, the more the linear spectral bias shows. The same applies to the spread. The logarithmic unit is in semitones. To convert, etiher divide by 12 to get the octave of one standard deviation, or divide by 6 to get the width of the filter in octaves. One clear observation is that the width is now in a range that scales with what we hear, growing fourfold as the filter goes from resonanting to more broadband. +code:: -STRONG::A few notes on the impact of the scale options:: - -CODE:: -// The computation of the centroids and other moments can also be done considering a logarithmic pitch scale, and/or the power of the magnitudes. This yields values that are more in line with the expectation of the users of equalisers for instance, where the shape is often drawn and described in logairhmic terms, i.e. dB per octave. - -// For instance, compare the values of the centroid and the spread in both scales: -( -{ - var source = BPF.ar(PinkNoise.ar(0.1),MouseX.kr().exprange(300,3000).poll(1,label: "filter frequency"), 0.5); - FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0)[0].lag.poll(1,"linear centroid"); - FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1)[0].lag.midicps.poll(1,"exponential centroid");//convert from midi to Hz - source.dup -}.play -) - -// The lower one gets in frequency, the more the linear spectral bias shows. The same applies to the spread: - -( -{ - var source = BPF.ar(PinkNoise.ar(0.1),440, MouseX.kr().exprange(0.1,4).poll(1,label: "filter RQ")); - FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0)[1].lag.poll(1,"linear spread"); - FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1)[1].lag.poll(1,"exponential spread"); - source.dup -}.play -) - -// The logarythmic unit is in semitones. To convert, either divide by 12 to get the octave of one standard deviation, or divide by 6 to get the width of the filter in octaves. One clear observation is that the width is now in a range that scales with what we hear, growing fourfold as the filter goes from resonating to more broadband. - -// An example of productive mapping between filters parameters and logarithmic centroid values allows to make a simple automatic subtractive noise resynthesis - -// load a beat -b = Buffer.read(s,FluidFilesPath("Nicol-LoopE-M.wav")); - -//logcentroid version ( { - var source = PlayBuf.ar(1,b,loop: 1); - var loudness, centroid, spread; - - #centroid,spread = Lag.kr(FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 1, power: 1, hopSize: 128),128/SampleRate.ir); - loudness = Lag.kr(FluidLoudness.kr(source,hopSize: 128),128/SampleRate.ir); - [ - DelayN.ar(source,delaytime: 1024/SampleRate.ir), - BBandPass.ar(WhiteNoise.ar(), - centroid.midicps, - (spread/6), - loudness[0].dbamp * 2 - ) - ] + var freq = MouseX.kr(20,20000,1); + var q = MouseY.kr(1,100); + var noise = BPF.ar(PinkNoise.ar,freq,q.reciprocal); + var lin = FluidStats.kr(FluidSpectralShape.kr(noise,[\centroid,\spread]),40)[0]; + var log = FluidStats.kr(FluidSpectralShape.kr(noise,[\centroid,\spread],unit:1,power:1),40)[0]; + + freq.poll(label:"filter frequency "); + q.poll(label:"filter q "); + lin[0].poll(label:"lin centroid "); + lin[1].poll(label:"lin spread "); + log[0].midicps.poll(label:"log centroid "); + log[1].poll(label:"log spread "); + 0.poll(label:"-------------------------"); + + noise.dup; }.play; ) -//lincentroid version for comparison -( -{ - var source = PlayBuf.ar(1,b,loop: 1); - var loudness, centroid, spread; - - #centroid,spread = Lag.kr(FluidSpectralShape.kr(source, minFreq: 20, maxFreq: 20000, unit: 0, power: 0, hopSize: 128),128/SampleRate.ir); - loudness = Lag.kr(FluidLoudness.kr(source,hopSize: 128),128/SampleRate.ir); - [ - DelayN.ar(source,delaytime: 1024/SampleRate.ir), - Sanitize.ar(BBandPass.ar(WhiteNoise.ar(), - centroid, - (spread * 2/centroid).max(0.001), - loudness[0].dbamp * 2 - )) - ] -}.play; -) -:: +:: \ No newline at end of file diff --git a/example-code/sc/UMAP.scd b/example-code/sc/UMAP.scd index db7738fc..dbd83a0e 100644 --- a/example-code/sc/UMAP.scd +++ b/example-code/sc/UMAP.scd @@ -1,189 +1,137 @@ - code:: -//Preliminaries: we want some points, a couple of FluidDataSets, a FluidStandardize and a FluidUMAP -( -~raw = FluidDataSet(s); -~standardized = FluidDataSet(s); -~reduced = FluidDataSet(s); -~normalized = FluidDataSet(s); -~standardizer = FluidStandardize(s); -~normalizer = FluidNormalize(s, 0.05, 0.95); -~umap = FluidUMAP(s).numDimensions_(2).numNeighbours_(5).minDist_(0.2).iterations_(50).learnRate_(0.2); -) - - -// build a dataset of 400 points in 3D (colour in RGB) -~colours = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1)); -~raw.load(Dictionary.newFrom([\cols, 3, \data, ~colours])); - -// check the entries -~raw.print; - -//First standardize our DataSet, then apply the UMAP to get 2 dimensions, then normalise these 2 for drawing in the full window size +// Using UMAP we'll reduce this dataset from 26 dimensions (26 MFCC values) down to 2 dimensions so we can plot it ( -~standardizer.fitTransform(~raw,~standardized,action:{"Standardized".postln}); -~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln}); -~normalizer.fitTransform(~reduced,~normalized,action:{"Normalized Output".postln}); +~loader = FluidLoadFolder(FluidFilesPath()).play; // load the audio (all the files in the FluCoMa audio files folder!) +~slicePoints = Buffer.read(s,FluidFilesPath("../Data/flucoma_corpus_slices.wav")); // load the slice positions of the audio +~ds_mfcc = FluidDataSet(s).read(FluidFilesPath("../Data/flucoma_corpus_mfcc.json")); // load the pre-analyzed dataset ) -//we recover the reduced dataset -~normalized.dump{|x| ~normalizedDict = x["data"]}; - -~normalized.print -~normalizedDict.postln -//Visualise the 2D projection of our original 4D data +// reduce and plot +// try changing the numNeighbours and minDist to see how it affects the plot. ( -w = Window("a perspective", Rect(128, 64, 200, 200)); -w.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours[key.asSymbol].flat; - } - } -}; -w.refresh; -w.front; -) - -//play with parameters -~umap.numNeighbours_(10).minDist_(0.5).iterations_(100).learnRate_(0.1); - -//rerun the UMAP -~umap.fitTransform(~standardized,~reduced,action:{"Finished UMAP".postln}); - -//draw to compare -( -~normalizer.fitTransform(~reduced,~normalized,action:{ - "Normalized Output".postln; - ~normalized.dump{|x| - ~normalizedDict = x["data"]; - - { - u = Window("another perspective", Rect(328, 64, 200, 200)); - u.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours[key.asSymbol].flat; - }; +var reduced = FluidDataSet(s); +var tree; +FluidUMAP(s,2,numNeighbours:15,minDist:0.1).fitTransform(~ds_mfcc,reduced); +FluidNormalize(s).fitTransform(reduced,reduced);// normalize so it's easier to plot +tree = FluidKDTree(s).fit(reduced); // use a kdtree to find the point nearest to the mouse position +reduced.dump({ + arg dict; + var xybuf = Buffer.alloc(s,2); + defer{ + FluidPlotter(dict:dict,mouseMoveAction:{ + arg view, x, y; + xybuf.setn(0,[x,y]); + tree.kNearest(xybuf,1,{ + arg id; + var index = id.asInteger; + defer{ + view.highlight_(id); }; - }; - u.refresh; - u.front; - - }.defer; + { + var start = Index.kr(~slicePoints,index); + var end = Index.kr(~slicePoints,index+1); + var sig = PlayBuf.ar(2,~loader.buffer,BufRateScale.ir(~loader.buffer),1,start)[0]; + var dur_sec = min((end-start) / SampleRate.ir,1); + var env = EnvGen.kr(Env([0,1,1,0],[0.03,dur_sec-0.06,0.03]),doneAction:2); + sig.dup * env; + }.play; + }); + }); }; }); ) -// now run new random points on the same training material. Colours should be scattered around the same space - -~newDS = FluidDataSet(s); -~colours2 = Dictionary.newFrom(400.collect{|i|[("entry"++i).asSymbol, 3.collect{1.0.rand}]}.flatten(1)); -~newDS.load(Dictionary.newFrom([\cols, 3, \data, ~colours2])); - -//we need to standardize to the same space -~newDSstan = FluidDataSet(s); -~standardizer.transform(~newDS, ~newDSstan); - -//then we can run the umap -~newDSmap = FluidDataSet(s); -~umap.transform(~newDSstan, ~newDSmap); +:: +strong::Reducing a 3 dimensional colour space to 2 dimensional space:: +code:: -//then we can draw and look +// make a dataset of 100 random 3 dimensional points (that will later be RGB values) ( -~normalizer.transform(~newDSmap,~normalized,action:{ - "Normalized Output".postln; - ~normalized.dump{|x| - ~normalizedDict = x["data"]; - { - t = Window("new material", Rect(528, 64, 200, 200)); - t.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours2[key.asSymbol][0], ~colours2[key.asSymbol][1],~colours2[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours2[key.asSymbol].flat; - }; - }; - }; - t.refresh; - t.front; - }.defer; - }; -}); +~ds_rgb = FluidDataSet(s).load( + Dictionary.newFrom([ + "cols",3, + "data",Dictionary.newFrom( + 100.collect{ + arg i; + [i,{rrand(0.0,1.0)} ! 3] + }.flatten; + ) + ]) +); +~ds_rgb.print; ) -//if we process the original dataset, we will see small differences in positions -~reduced2 = FluidDataSet(s); -~umap.transform(~standardized, ~reduced2, action: {\done.postln;}); - -//then we can draw and look +// rather than trying to plot these points in 3 dimensional space, +// we'll reduce it to 2 dimensional space so it looks better on a screen +// again play around with the arguments numNeighbours and minDist to see +// how they affect he spread of the points ( -~normalizer.transform(~reduced2,~normalized,action:{ - "Normalized Output".postln; - ~normalized.dump{|x| - ~normalizedDict = x["data"]; - { - z = Window("old material", Rect(728, 64, 200, 200)); - z.drawFunc = { - Pen.use { - ~normalizedDict.keysValuesDo{|key, val| - Pen.fillColor = Color.new(~colours[key.asSymbol][0], ~colours[key.asSymbol][1],~colours[key.asSymbol][2]); - Pen.fillOval(Rect((val[0] * 200), (val[1] * 200), 5, 5)); - ~colours[key.asSymbol].flat; - }; - }; +var reduced = FluidDataSet(s); +FluidUMAP(s,2,numNeighbours:15,minDist:0.6).fitTransform(~ds_rgb,reduced); +FluidNormalize(s).fitTransform(reduced,reduced); +~ds_rgb.dump{ + arg rgb; + reduced.dump{ + arg xy; + defer{ + var fp = FluidPlotter(dict:xy).pointSizeScale_(2); + rgb["data"].keysValuesDo{ + arg k, v; + fp.pointColor_(k,Color(*v)); }; - z.refresh; - z.front; - }.defer; + }; }; -}); +}; ) -//this is because the fitTransform method has the advantage of being certain that the data it transforms is the one that has been used to fit the model. This allows for more accurate distance measurement. +:: +strong::Retrieving values on the server:: +code:: -//to check, let's retrieve a single point and predict its position +// Using UMAP we'll reduce this dataset from 26 dimensions (26 MFCC values) down to 2 dimensions +// wait for it to finish ( -~sourcePoint = Buffer(s); -~original = Buffer(s); -~standed = Buffer(s); -~umaped = Buffer(s); +~loader = FluidLoadFolder(FluidFilesPath()).play; // load the audio (all the files in the FluCoMa audio files folder!) +~slicePoints = Buffer.read(s,FluidFilesPath("../Data/flucoma_corpus_slices.wav")); // load the slice positions of the audio +~ds_mfcc = FluidDataSet(s).read(FluidFilesPath("../Data/flucoma_corpus_mfcc.json")); // load the pre-analyzed dataset +~reduced = FluidDataSet(s); +~umap = FluidUMAP(s,2,15,0.1).fitTransform(~ds_mfcc,~reduced); +~normalizer = FluidNormalize(s).fitTransform(~reduced,~reduced,action:{"done".postln;}); ) -//retrieve the 3D original -~raw.getPoint("entry49",~sourcePoint) -//retrieve the fitTransformed point as the most accurate point -~reduced.getPoint("entry49",~original, {~original.getn(0,2,{|x|x.postln})}) -//retreive the transformed point, via the standardizer -~standardizer.transformPoint(~sourcePoint,~standed); -~umap.transformPoint(~standed, ~umaped, {~umaped.getn(0,2,{|x|x.postln})}) - -// one can also retrieve in control rate with Server Side Queries -// Let's map our learned UMAP dimensions to the controls of a processor - ( -{ - var trig = Impulse.kr(1); - var point = WhiteNoise.kr(1.dup(3)); - var inputPoint = LocalBuf(3); - var standPoint = LocalBuf(3); - var outputPoint = LocalBuf(2); - var cue1, cue2; - Poll.kr(trig, point, [\pointX,\pointY,\pointZ]); - point.collect{ |p,i| BufWr.kr([p],inputPoint,i)}; - cue1 = ~standardizer.kr(trig,inputPoint,standPoint); - Poll.kr(cue1,BufRd.kr(1,standPoint,(0..2),interpolation:0),[\stdX,\stdY, \stdZ]); - cue2 = ~umap.kr(cue1, standPoint, outputPoint); - Poll.kr(cue2,BufRd.kr(1,outputPoint,[0,1],interpolation:0),[\newDimA,\newDimB]); - Silent.ar; -}.play; +~reduced.dump{ + arg dict; + defer{ + ~fp = FluidPlotter(dict:dict).addPoint_("current",0.5,0.5,Color.red,2); + + { + // play the original buffer back, but backwardsa and a little faster for some variety + var sig = PlayBuf.ar(2,~loader.buffer,-1.3,1,rrand(0,~loader.buffer.numFrames-1),1)[0]; + var mfccs = FluidMFCC.kr(sig,26,startCoeff:1); // get mfcc analyses + var mfccbuf = LocalBuf(mfccs.numChannels); + var reducedbuf = LocalBuf(2); + var normbuf = LocalBuf(2); + var trig = Impulse.kr(30); + var xy; + FluidKrToBuf.kr(mfccs,mfccbuf); + ~umap.kr(trig,mfccbuf,reducedbuf); // project the mfcc analyses into the umap space + ~normalizer.kr(trig,reducedbuf,normbuf); // make sure it's in the normalized space + xy = FluidBufToKr.kr(normbuf); + SendReply.kr(trig,"/xy",xy); // send back to the language for plotting + sig; + }.play; + + OSCdef(\xy,{ + arg msg; + defer{ + ~fp.setPoint_("current",msg[3],msg[4],Color.red,2); // shows where the current mfcc analysis would be + }; + },"/xy"); + } +}; ) :: diff --git a/render-flucoma-docs-only.scd b/render-flucoma-docs-only.scd index 9ba79450..01bfe58b 100644 --- a/render-flucoma-docs-only.scd +++ b/render-flucoma-docs-only.scd @@ -16,6 +16,8 @@ fork({ },AppClock); ) +// also check me! +FluidCorpusManipulation // also check me! FluidManipulationClient