diff --git a/src/funkin/vis/AudioBuffer.hx b/src/funkin/vis/AudioBuffer.hx deleted file mode 100644 index fa3e261..0000000 --- a/src/funkin/vis/AudioBuffer.hx +++ /dev/null @@ -1,15 +0,0 @@ -package funkin.vis; - -import lime.utils.UInt16Array; - -class AudioBuffer -{ - public var data(default, null):UInt16Array; - public var sampleRate(default, null):Float; - - public function new(data:UInt16Array, sampleRate:Float) - { - this.data = data; - this.sampleRate = sampleRate; - } -} \ No newline at end of file diff --git a/src/funkin/vis/AudioClip.hx b/src/funkin/vis/AudioClip.hx index 0b4e56b..26924da 100644 --- a/src/funkin/vis/AudioClip.hx +++ b/src/funkin/vis/AudioClip.hx @@ -5,7 +5,7 @@ package funkin.vis; */ interface AudioClip { - public var audioBuffer(default, null):AudioBuffer; - public var currentFrame(get, never):Int; - public var source:Dynamic; + public var channels(get, never):Int; + public var sampleRate(get, never):Int; + public function getTimeDomainData(fftN:Int):Array; } \ No newline at end of file diff --git a/src/funkin/vis/_internal/html5/AnalyzerNode.hx b/src/funkin/vis/_internal/html5/AnalyzerNode.hx index 34c735e..3385d61 100644 --- a/src/funkin/vis/_internal/html5/AnalyzerNode.hx +++ b/src/funkin/vis/_internal/html5/AnalyzerNode.hx @@ -1,9 +1,13 @@ package funkin.vis._internal.html5; -import funkin.vis.AudioBuffer; +import lime.utils.Float32Array; +import funkin.vis.audioclip.frontends.LimeAudioClip; + #if lime_howlerjs import lime.media.howlerjs.Howl; -import js.html.audio.AnalyserNode as AnalyseWebAudio; +import js.html.audio.AudioNode; +import js.html.audio.AnalyserNode as JSAnalyserNode; +import js.html.audio.BaseAudioContext; #end // note: analyze and analyse are both correct spellings of the word, @@ -11,41 +15,91 @@ import js.html.audio.AnalyserNode as AnalyseWebAudio; // and we use the Z variant here... class AnalyzerNode { - #if lime_howlerjs - public var analyzer:AnalyseWebAudio; + public var analyser:JSAnalyserNode; public var maxDecibels:Float = -30; public var minDecibels:Float = -100; public var fftSize:Int = 2048; + + public var limeAudioClip:LimeAudioClip; + + var audioNode:AudioNode; + var array:Float32Array; #end // #region yoooo - public function new(?audioClip:AudioClip) + public function new(?limeAudioClip:LimeAudioClip) { - trace("Loading audioClip"); - #if lime_howlerjs - analyzer = new AnalyseWebAudio(audioClip.source._sounds[0]._node.context); - audioClip.source._sounds[0]._node.connect(analyzer); - // trace(audioClip.source._sounds[0]._node.context.sampleRate); - // trace(analyzer); - // trace(analyzer.fftSize); - // howler = cast buffer.source; - // trace(howler); - getFloatFrequencyData(); - #end + this.limeAudioClip = limeAudioClip; + #end } - public function getFloatFrequencyData():Array + public function getFloatFrequencyData():Float32Array { #if lime_howlerjs - var array:js.lib.Float32Array = new js.lib.Float32Array(analyzer.frequencyBinCount); - analyzer.fftSize = fftSize; - analyzer.minDecibels = minDecibels; - analyzer.maxDecibels = maxDecibels; - analyzer.getFloatFrequencyData(array); - return cast array; + var desiredLength = fftSize >> 1; + if (array == null || array.length != desiredLength) array = new Float32Array(desiredLength); + + // really a crime to read this code but this gets the audioNode. + var previousAudioNode = audioNode; + @:privateAccess + { + #if lime_funkin + var howlSound = limeAudioClip.audioSource.__backend.howlSound; + if (howlSound != null) + #else + var howl = limeAudioClip.audioSource.buffer.__srcHowl; + if (howl == null) return cast array; + + var howlSound = untyped limeAudioClip.audioSource.buffer.__srcHowl._soundById(limeAudioClip.audioSource.__backend.id); + if (!(untyped howlSound)) howlSound = untyped limeAudioClip.audioSource.buffer.__srcHowl._sounds[0]; + + if (untyped howlSound) + #end + { + if (untyped howlSound._node) + { + if (untyped howlSound._node.bufferSource) + { + audioNode = untyped howlSound._node.bufferSource; + } + else + { + audioNode = untyped howlSound._node; + } + } + else + { + audioNode = null; + } + } + else + { + audioNode = null; + } + } + + if (audioNode == null) return cast array; + + if (previousAudioNode != audioNode) + { + var context:BaseAudioContext = audioNode.context; + if (analyser == null || context != analyser.context) analyser = new JSAnalyserNode(context); + else analyser.disconnect(); + + audioNode.connect(analyser); + } + + analyser.smoothingTimeConstant = 0.1; + analyser.fftSize = fftSize; + analyser.maxDecibels = maxDecibels; + analyser.minDecibels = minDecibels; + + analyser.getFloatFrequencyData(array); + return array; + #else + return new Float32Array(0); #end - return []; } } \ No newline at end of file diff --git a/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx b/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx index 46d584f..8111643 100644 --- a/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx +++ b/src/funkin/vis/audioclip/frontends/LimeAudioClip.hx @@ -2,8 +2,15 @@ package funkin.vis.audioclip.frontends; import flixel.FlxG; import flixel.math.FlxMath; -import funkin.vis.AudioBuffer; +import lime.media.AudioContextType; import lime.media.AudioSource; +import lime.media.AudioManager; + +#if lime_funkin +import lime.utils.Float32Array; +#elseif !js +import lime.utils.ArrayBufferView.ArrayBufferIO; +#end /** * Implementation of AudioClip for Lime. @@ -17,39 +24,117 @@ import lime.media.AudioSource; */ class LimeAudioClip implements funkin.vis.AudioClip { - public var audioBuffer(default, null):AudioBuffer; - public var currentFrame(get, never):Int; - public var source:Dynamic; + public var audioSource:AudioSource; + public var channels(get, never):Int; + public var sampleRate(get, never):Int; public function new(audioSource:AudioSource) { - var data:lime.utils.UInt16Array = cast audioSource.buffer.data; + this.audioSource = audioSource; + } - #if web - var sampleRate:Float = audioSource.buffer.src._sounds[0]._node.context.sampleRate; + inline function get_channels():Int + { + #if (js && html5 && howlerjs) + if (audioSource.buffer != null && audioSource.buffer.channels > 0) return audioSource.buffer.channels; + return 2; #else - var sampleRate = audioSource.buffer.sampleRate; + return audioSource.buffer != null ? audioSource.buffer.channels : 0; #end - - this.audioBuffer = new AudioBuffer(data, sampleRate); - this.source = audioSource.buffer.src; } - private function get_currentFrame():Int + inline function get_sampleRate():Int { - var dataLength:Int = 0; - - #if web - dataLength = source.length; + #if (js && html5 && howlerjs) + if (audioSource.buffer != null && audioSource.buffer.sampleRate > 0) return audioSource.buffer.sampleRate; + else if (AudioManager.context != null && AudioManager.context.type == AudioContextType.WEB) return Std.int(AudioManager.context.web.sampleRate); + return 44100; #else - dataLength = audioBuffer.data.length; + return audioSource.buffer != null ? audioSource.buffer.sampleRate : 0; #end + } + + // Prevents a memory leak by reusing array + var _buffer:Array = []; + + #if lime_funkin + var _array:Float32Array; + + public function getTimeDomainData(fftN:Int):Array + { + // Allocate it first! + if (fftN > _buffer.length) _buffer.resize(fftN); + + if (_array == null || fftN > _array.length) _array = new Float32Array(fftN); + + var len = audioSource.getFloatTimeDomainData(_array, fftN); - var value = Std.int(FlxMath.remapToRange(FlxG.sound.music.time, 0, FlxG.sound.music.length, 0, dataLength)); + for (i in 0...len) _buffer[i] = _array[i]; + for (i in len...fftN) _buffer[i] = 0; + return _buffer; + } + #elseif !js + public function getTimeDomainData(fftN:Int):Array + { + // Allocate it first! + if (fftN > _buffer.length) _buffer.resize(fftN); + + inline function empty() + { + for (i in 0...fftN) _buffer[i] = 0; + return _buffer; + } + + var audioBuffer = audioSource.buffer; + if (audioBuffer == null) return empty(); + + var byteRate:Int = audioBuffer.bitsPerSample >> 3; + var currentFrame:Int = (Std.int(audioSource.currentTime / 1000 * audioBuffer.sampleRate) - fftN) * audioBuffer.channels * byteRate; + + if (currentFrame <= 0) return empty(); - if (value < 0) - return -1; + var valueSize:Int = 1 << (audioBuffer.bitsPerSample - 1); + var buffer = audioBuffer.data.buffer, pos = 0, v = 0, c = 0; - return value; + inline function idiv(num:Int, denom:Int):Int + { + return #if (cpp && !cppia) cpp.NativeMath.idiv(num, denom) #else Std.int(num / denom) #end; + } + + while (pos < fftN) + { + switch (byteRate) + { + case 2: v = idiv(ArrayBufferIO.getInt16(buffer, currentFrame), audioBuffer.channels); + case 3: + v = ArrayBufferIO.getUint16(buffer, currentFrame) | (buffer.get(currentFrame + 2) << 16); + if (v & 0x800000 != 0) v -= 0x1000000; + v = idiv(v, audioBuffer.channels); + case 4: v = idiv(ArrayBufferIO.getInt32(buffer, currentFrame), audioBuffer.channels); + default: v = idiv(ArrayBufferIO.getInt8(buffer, currentFrame), audioBuffer.channels); + } + + c++; + if (c == audioBuffer.channels) + { + _buffer[pos++] = v / valueSize; + c = v = 0; + } + + currentFrame += byteRate; + if (currentFrame >= buffer.length) + { + for (i in pos...fftN) _buffer[i] = 0; + break; + } + } + + return _buffer; + } + #else + public function getTimeDomainData(fftN:Int):Array + { + throw "Your not supposed to use this in js!"; } + #end } diff --git a/src/funkin/vis/dsp/SpectralAnalyzer.hx b/src/funkin/vis/dsp/SpectralAnalyzer.hx index 45c3c87..1653e27 100644 --- a/src/funkin/vis/dsp/SpectralAnalyzer.hx +++ b/src/funkin/vis/dsp/SpectralAnalyzer.hx @@ -2,12 +2,15 @@ package funkin.vis.dsp; import flixel.FlxG; import flixel.math.FlxMath; -import funkin.vis._internal.html5.AnalyzerNode; import funkin.vis.audioclip.frontends.LimeAudioClip; import grig.audio.FFT; import grig.audio.FFTVisualization; import lime.media.AudioSource; +#if web +import funkin.vis._internal.html5.AnalyzerNode; +#end + using grig.audio.lime.UInt8ArrayTools; typedef Bar = @@ -41,7 +44,7 @@ class SpectralAnalyzer public var minFreq:Float = 50; public var maxFreq:Float = 22000; // Awkwardly, we'll have to interfaces for now because there's too much platform specific stuff we need - private var audioSource:AudioSource; + private var audioSource(get, set):AudioSource; private var audioClip:AudioClip; private var barCount:Int; private var maxDelta:Float; @@ -61,7 +64,7 @@ class SpectralAnalyzer private function freqToBin(freq:Float, mathType:MathType = Round):Int { - var bin = freq * fftN2 / audioClip.audioBuffer.sampleRate; + var bin = freq * fftN2 / audioClip.sampleRate; return switch (mathType) { case Round: Math.round(bin); case Floor: Math.floor(bin); @@ -145,14 +148,13 @@ class SpectralAnalyzer public function new(audioSource:AudioSource, barCount:Int, maxDelta:Float = 0.01, peakHold:Int = 30) { - this.audioSource = audioSource; this.audioClip = new LimeAudioClip(audioSource); this.barCount = barCount; this.maxDelta = maxDelta; this.peakHold = peakHold; #if web - htmlAnalyzer = new AnalyzerNode(audioClip); + htmlAnalyzer = new AnalyzerNode(cast audioClip); #else fft = new FFT(fftN); #end @@ -165,16 +167,19 @@ class SpectralAnalyzer { if(levels == null) levels = new Array(); #if web - var amplitudes:Array = htmlAnalyzer.getFloatFrequencyData(); + var amplitudes:lime.utils.Float32Array = htmlAnalyzer.getFloatFrequencyData(); + untyped console.log(amplitudes); for (i in 0...bars.length) { var bar = bars[i]; var binLo = bar.binLo; var binHi = bar.binHi; + trace(bar, binLo, binHi); var value:Float = minDb; for (j in (binLo + 1)...(binHi)) { value = Math.max(value, amplitudes[Std.int(j)]); + trace(value); } // this isn't for clamping, it's to get a value @@ -193,35 +198,13 @@ class SpectralAnalyzer return levels; #else - var numOctets = Std.int(audioSource.buffer.bitsPerSample / 8); - var wantedLength = fftN * numOctets * audioSource.buffer.channels; - var startFrame = audioClip.currentFrame; + var buffer = audioClip.getTimeDomainData(fftN); - if (startFrame < 0) + for (i in 0...fftN) { - return levels = [for (bar in 0...barCount) {value: 0, peak: 0}]; + _mixedCache[i] = buffer[i] * blackmanWindow[i]; } - startFrame -= startFrame % numOctets; - var segment = audioSource.buffer.data.subarray(startFrame, min(startFrame + wantedLength, audioSource.buffer.data.length)); - - getSignal(segment, audioSource.buffer.bitsPerSample); // Sets _buffer - - if (audioSource.buffer.channels > 1) { - var wantedArrayLength = Std.int(_buffer.length / audioSource.buffer.channels); - if (_mixedCache.length != wantedArrayLength) - _mixedCache.resize(wantedArrayLength); - - for (i in 0..._mixedCache.length) { - _mixedCache[i] = 0.0; - for (c in 0...audioSource.buffer.channels) { - _mixedCache[i] += 0.7 * _buffer[i*audioSource.buffer.channels+c]; - } - _mixedCache[i] *= blackmanWindow[i]; - } - //_buffer = _mixedCache; - } - var range = 16; var freqs = fft.calcFreq(_mixedCache); @@ -345,4 +328,14 @@ class SpectralAnalyzer resizeBlackmanWindow(fftN); return pow2; } + + function get_audioSource():AudioSource + { + return if (audioClip is LimeAudioClip) (cast audioClip :LimeAudioClip).audioSource; else null; + } + + function set_audioSource(value:AudioSource):AudioSource + { + return if (audioClip is LimeAudioClip) (cast audioClip :LimeAudioClip).audioSource = value; else value; + } }