From ec222bd67d58b9bd587f9daed766d6482a346a54 Mon Sep 17 00:00:00 2001 From: "5684185+vsariola@users.noreply.github.com" <5684185+vsariola@users.noreply.github.com> Date: Sat, 2 Nov 2024 15:04:19 +0200 Subject: [PATCH] feat(tracker): oscilloscope and LUFS / true peak detection In addition to the oscilloscope and loudness/peak detections, this commit refactors all the channels between components (i.e. ModelMessages and PlayerMessages) etc. into a new class Broker. This was done because now we have one more goroutine running: a Detector, where the loudness / true peak detection is done in another thread. The different threads/components are only aware of the Broker and communicate through it. Currently, it's just a collection of channels, so it's many-to-one communication, but in the future, we could change Broker to have many-to-one-to-many communication. Related to #61 --- CHANGELOG.md | 4 + cmd/sointu-track/main.go | 5 +- cmd/sointu-vsti/main.go | 5 +- go.mod | 11 +- go.sum | 36 ++-- tracker/broker.go | 98 +++++++++ tracker/detector.go | 391 ++++++++++++++++++++++++++++++++++ tracker/gioui/oscilloscope.go | 183 ++++++++++++++++ tracker/gioui/songpanel.go | 18 +- tracker/gioui/tracker.go | 5 +- tracker/gioui/vumeter.go | 10 +- tracker/model.go | 68 +++--- tracker/model_test.go | 6 +- tracker/player.go | 70 +++--- tracker/scopemodel.go | 141 ++++++++++++ tracker/volume.go | 68 ------ 16 files changed, 945 insertions(+), 174 deletions(-) create mode 100644 tracker/broker.go create mode 100644 tracker/detector.go create mode 100644 tracker/gioui/oscilloscope.go create mode 100644 tracker/scopemodel.go delete mode 100644 tracker/volume.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d57298a..b4b30a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +- The loudness detection is now LUFS and peak detection is based on oversampled + true peak detection +- Oscilloscope to visualize the outputted waveform ([#61][i61]) - Toggle button to keep instruments and tracks linked, and buttons to to split instruments and tracks with more than 1 voice into parallel ones ([#163][i163], [#157][i157]) @@ -245,6 +248,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). [0.3.0]: https://github.com/vsariola/sointu/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/vsariola/sointu/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/vsariola/sointu/compare/4klang-3.11...v0.1.0 +[i61]: https://github.com/vsariola/sointu/issues/61 [i65]: https://github.com/vsariola/sointu/issues/65 [i68]: https://github.com/vsariola/sointu/issues/68 [i77]: https://github.com/vsariola/sointu/issues/77 diff --git a/cmd/sointu-track/main.go b/cmd/sointu-track/main.go index f2d410f..d8354b2 100644 --- a/cmd/sointu-track/main.go +++ b/cmd/sointu-track/main.go @@ -47,7 +47,10 @@ func main() { midiContext := gomidi.NewContext() defer midiContext.Close() midiContext.TryToOpenBy(*defaultMidiInput, *firstMidiInput) - model, player := tracker.NewModelPlayer(cmd.MainSynther, midiContext, recoveryFile) + broker := tracker.NewBroker() + model, player := tracker.NewModelPlayer(broker, cmd.MainSynther, midiContext, recoveryFile) + detector := tracker.NewDetector(broker) + go detector.Run() if a := flag.Args(); len(a) > 0 { f, err := os.Open(a[0]) diff --git a/cmd/sointu-vsti/main.go b/cmd/sointu-vsti/main.go index 009db67..cce5d86 100644 --- a/cmd/sointu-vsti/main.go +++ b/cmd/sointu-vsti/main.go @@ -72,7 +72,10 @@ func init() { rand.Read(randBytes) recoveryFile = filepath.Join(configDir, "sointu", "sointu-vsti-recovery-"+hex.EncodeToString(randBytes)) } - model, player := tracker.NewModelPlayer(cmd.MainSynther, NullMIDIContext{}, recoveryFile) + broker := tracker.NewBroker() + model, player := tracker.NewModelPlayer(broker, cmd.MainSynther, NullMIDIContext{}, recoveryFile) + detector := tracker.NewDetector(broker) + go detector.Run() t := gioui.NewTracker(model) model.InstrEnlarged().Bool().Set(true) diff --git a/go.mod b/go.mod index cd6c499..5a9a8f9 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,12 @@ require ( gioui.org/x v0.7.1 github.com/Masterminds/sprig v2.22.0+incompatible github.com/ebitengine/oto/v3 v3.3.0 + github.com/viterin/vek v0.4.2 gitlab.com/gomidi/midi/v2 v2.2.10 golang.org/x/exp/shiny v0.0.0-20240707233637-46b078467d37 golang.org/x/text v0.16.0 gopkg.in/yaml.v2 v2.3.0 - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 + gopkg.in/yaml.v3 v3.0.1 pipelined.dev/audio/vst2 v0.10.1-0.20240223162706-41e9b65fb5c2 ) @@ -21,19 +22,21 @@ require ( git.wow.st/gmp/jni v0.0.0-20210610011705-34026c7e22d0 // indirect github.com/Masterminds/goutils v1.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect + github.com/chewxy/math32 v1.11.1 // indirect github.com/ebitengine/purego v0.8.0 // indirect github.com/go-text/typesetting v0.1.1 // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/google/uuid v1.1.2 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.11 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect - github.com/stretchr/testify v1.6.1 // indirect + github.com/viterin/partial v1.1.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect - golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/image v0.18.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect pipelined.dev/pipe v0.11.0 // indirect pipelined.dev/signal v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 5d0f694..3083e3c 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,11 @@ github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3Q github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/chewxy/math32 v1.11.1 h1:b7PGHlp8KjylDoU8RrcEsRuGZhJuz8haxnKfuMMRqy8= +github.com/chewxy/math32 v1.11.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ebitengine/oto/v3 v3.3.0 h1:34lJpJLqda0Iee9g9p8RWtVVwBcOOO2YSIS2x4yD1OQ= github.com/ebitengine/oto/v3 v3.3.0/go.mod h1:MZeb/lwoC4DCOdiTIxYezrURTw7EvK/yF863+tmBI+U= github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE= @@ -35,36 +38,43 @@ github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/viterin/partial v1.1.0 h1:iH1l1xqBlapXsYzADS1dcbizg3iQUKTU1rbwkHv/80E= +github.com/viterin/partial v1.1.0/go.mod h1:oKGAo7/wylWkJTLrWX8n+f4aDPtQMQ6VG4dd2qur5QA= +github.com/viterin/vek v0.4.2 h1:Vyv04UjQT6gcjEFX82AS9ocgNbAJqsHviheIBdPlv5U= +github.com/viterin/vek v0.4.2/go.mod h1:A4JRAe8OvbhdzBL5ofzjBS0J29FyUrf95tQogvtHHUc= gitlab.com/gomidi/midi/v2 v2.2.10 h1:u9D+5TM0vkFWF5DcO6xGKG99ERYqksh6wPj2X2Rx5A8= gitlab.com/gomidi/midi/v2 v2.2.10/go.mod h1:ENtYaJPOwb2N+y7ihv/L7R4GtWjbknouhIIkMrJ5C0g= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/exp/shiny v0.0.0-20240707233637-46b078467d37 h1:SOSg7+sueresE4IbmmGM60GmlIys+zNX63d6/J4CMtU= golang.org/x/exp/shiny v0.0.0-20240707233637-46b078467d37/go.mod h1:3F+MieQB7dRYLTmnncoFbb1crS5lfQoTfDgQy6K4N0o= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= pipelined.dev/audio/vst2 v0.10.1-0.20240223162706-41e9b65fb5c2 h1:qrI7YY5ZH4pJflMfzum2TKvA1NaX+H4feaA6jweX2R8= pipelined.dev/audio/vst2 v0.10.1-0.20240223162706-41e9b65fb5c2/go.mod h1:wETLxsbBPftj6t4iVBCXvH/Xgd27ZgIC4hNnHDYNuz8= pipelined.dev/pipe v0.10.0/go.mod h1:aIt+NPlW0QLYByqYniG77lTxSvl7OtCNLws/m+Xz5ww= diff --git a/tracker/broker.go b/tracker/broker.go new file mode 100644 index 0000000..c5fbede --- /dev/null +++ b/tracker/broker.go @@ -0,0 +1,98 @@ +package tracker + +import ( + "sync" + + "github.com/vsariola/sointu" + "github.com/vsariola/sointu/vm" +) + +type ( + // Broker is the centralized message broker for the tracker. It is used to + // communicate between the player, the model, and the loudness detector. At + // the moment, the broker is just many-to-one communication, implemented + // with one channel for each recipient. Additionally, the broker has a + // sync.pool for *sointu.AudioBuffers, from which the player can get and + // return buffers to pass buffers around without allocating new memory every + // time. We can later consider making many-to-many types of communication + // and more complex routing logic to the Broker if needed. + Broker struct { + ToModel chan MsgToModel + ToPlayer chan any // TODO: consider using a sum type here, for a bit more type safety. See: https://www.jerf.org/iri/post/2917/ + ToDetector chan MsgToDetector + + bufferPool sync.Pool + } + + // MsgToModel is a message sent to the model. The most often sent data + // (Panic, SongPosition, VoiceLevels and DetectorResult) are not boxed to + // avoid allocations. All the infrequently passed messages can be boxed & + // cast to any; casting pointer types to any is cheap (does not allocate). + MsgToModel struct { + HasPanicPosLevels bool + Panic bool + SongPosition sointu.SongPos + VoiceLevels [vm.MAX_VOICES]float32 + + HasDetectorResult bool + DetectorResult DetectorResult + + TriggerChannel int // note: 0 = no trigger, 1 = first channel, etc. + Reset bool // true: playing started, so should reset the detector and the scope cursor + + Data any // TODO: consider using a sum type here, for a bit more type safety. See: https://www.jerf.org/iri/post/2917/ + } + + // MsgToDetector is a message sent to the detector. It contains a reset flag + // and a data field. The data field can contain many different messages, + // including *sointu.AudioBuffer for the detector to analyze and func() + // which gets executed in the detector goroutine. + MsgToDetector struct { + Reset bool + Data any // TODO: consider using a sum type here, for a bit more type safety. See: https://www.jerf.org/iri/post/2917/ + } +) + +func NewBroker() *Broker { + return &Broker{ + ToPlayer: make(chan interface{}, 1024), + ToModel: make(chan MsgToModel, 1024), + ToDetector: make(chan MsgToDetector, 1024), + bufferPool: sync.Pool{New: func() interface{} { return &sointu.AudioBuffer{} }}, + } +} + +func (b *Broker) Close() { + close(b.ToPlayer) + close(b.ToModel) + close(b.ToDetector) +} + +// GetAudioBuffer returns an audio buffer from the buffer pool. The buffer is +// guaranteed to be empty. After using the buffer, it should be returned to the +// pool with PutAudioBuffer. +func (b *Broker) GetAudioBuffer() *sointu.AudioBuffer { + return b.bufferPool.Get().(*sointu.AudioBuffer) +} + +// PutAudioBuffer returns an audio buffer to the buffer pool. If the buffer is +// not empty, its length is resetted (but capacity kept) before returning it to +// the pool. +func (b *Broker) PutAudioBuffer(buf *sointu.AudioBuffer) { + if len(*buf) > 0 { + *buf = (*buf)[:0] + } + b.bufferPool.Put(buf) +} + +// trySend is a helper function to send a value to a channel if it is not full. +// It is guaranteed to be non-blocking. Return true if the value was sent, false +// otherwise. +func trySend[T any](c chan<- T, v T) bool { + select { + case c <- v: + default: + return false + } + return true +} diff --git a/tracker/detector.go b/tracker/detector.go new file mode 100644 index 0000000..99b7b3b --- /dev/null +++ b/tracker/detector.go @@ -0,0 +1,391 @@ +package tracker + +import ( + "math" + + "github.com/viterin/vek/vek32" + "github.com/vsariola/sointu" +) + +type ( + Detector struct { + broker *Broker + loudnessDetector loudnessDetector + peakDetector peakDetector + } + + WeightingType int + LoudnessType int + PeakType int + + Decibel float32 + + LoudnessResult [NumLoudnessTypes]Decibel + PeakResult [NumPeakTypes][2]Decibel + + DetectorResult struct { + Loudness LoudnessResult + Peaks PeakResult + } + + loudnessDetector struct { + weighting weighting + states [2][3]biquadState + powers [2]RingBuffer[float32] // 0 = momentary, 1 = short-term + averagedPowers [2][]float32 + maxPowers [2]float32 + integratedPower float32 + tmp, tmp2 []float32 + tmpbool []bool + } + + biquadState struct { + x1, x2, y1, y2 float32 + } + + biquadCoeff struct { + b0, b1, b2, a1, a2 float32 + } + + weighting struct { + coeffs []biquadCoeff + offset float32 + } + + peakDetector struct { + oversampling bool + states [2]oversamplerState + windows [2][2]RingBuffer[float32] + maxPower [2]float32 + tmp, tmp2 []float32 + } + + oversamplerState struct { + history [11]float32 + tmp, tmp2 []float32 + } +) + +const ( + LoudnessMomentary LoudnessType = iota + LoudnessShortTerm + LoudnessMaxMomentary + LoudnessMaxShortTerm + LoudnessIntegrated + NumLoudnessTypes +) + +const MAX_INTEGRATED_DATA = 10 * 60 * 60 // 1 hour of samples at 10 Hz (100 ms per sample) + +const ( + PeakMomentary PeakType = iota + PeakShortTerm + PeakIntegrated + NumPeakTypes +) + +const ( + KWeighting WeightingType = iota + AWeighting + CWeighting + NoWeighting +) + +func NewDetector(b *Broker) *Detector { + return &Detector{ + broker: b, + loudnessDetector: makeLoudnessDetector(KWeighting), + peakDetector: makePeakDetector(true), + } +} + +func (s *Detector) Run() { + var chunkHistory sointu.AudioBuffer + for msg := range s.broker.ToDetector { + if msg.Reset { + s.loudnessDetector.reset() + s.peakDetector.reset() + } + switch data := msg.Data.(type) { + case *sointu.AudioBuffer: + buf := *data + for { + var chunk sointu.AudioBuffer + if len(chunkHistory) > 0 && len(chunkHistory) < 4410 { + l := min(len(buf), 4410-len(chunkHistory)) + chunkHistory = append(chunkHistory, buf[:l]...) + if len(chunkHistory) < 4410 { + break + } + chunk = chunkHistory + buf = buf[l:] + } else { + if len(buf) >= 4410 { + chunk = buf[:4410] + buf = buf[4410:] + } else { + chunkHistory = chunkHistory[:0] + chunkHistory = append(chunkHistory, buf...) + break + } + } + trySend(s.broker.ToModel, MsgToModel{ + HasDetectorResult: true, + DetectorResult: DetectorResult{ + Loudness: s.loudnessDetector.update(chunk), + Peaks: s.peakDetector.update(chunk), + }, + }) + } + s.broker.PutAudioBuffer(data) + case func(): + data() + } + } +} + +func makeLoudnessDetector(weighting WeightingType) loudnessDetector { + return loudnessDetector{ + weighting: weightings[weighting], + powers: [2]RingBuffer[float32]{ + {Buffer: make([]float32, 4)}, // momentary loudness + {Buffer: make([]float32, 30)}, // short-term loudness + }, + } +} + +func makePeakDetector(oversampling bool) peakDetector { + return peakDetector{ + oversampling: oversampling, + windows: [2][2]RingBuffer[float32]{ + {{Buffer: make([]float32, 4)}, {Buffer: make([]float32, 30)}}, // momentary and short-term peaks for left channel + {{Buffer: make([]float32, 4)}, {Buffer: make([]float32, 30)}}, // momentary and short-term peaks for right channel + }, + } +} + +/* +From matlab: +f = getFilter(weightingFilter('A-weighting','SampleRate',44100)); f.Numerator, f.Denominator +for i = 1:size(f.Numerator,1); fprintf("b0: %.16f, b1: %.16f, b2: %.16f, a1: %.16f, a2: %.16f\n",f.Numerator(i,:),f.Denominator(i,2:end)); end +f = getFilter(weightingFilter('C-weighting','SampleRate',44100)); f.Numerator, f.Denominator +for i = 1:size(f.Numerator,1); fprintf("b0: %.16f, b1: %.16f, b2: %.16f, a1: %.16f, a2: %.16f\n",f.Numerator(i,:),f.Denominator(i,2:end)); end +f = getFilter(weightingFilter('k-weighting','SampleRate',44100)); f.Numerator, f.Denominator +for i = 1:size(f.Numerator,1); fprintf("b0: %.16f, b1: %.16f, b2: %.16f, a1: %.16f, a2: %.16f\n",f.Numerator(i,:),f.Denominator(i,2:end)); end +*/ +var weightings = map[WeightingType]weighting{ + AWeighting: {coeffs: []biquadCoeff{ + {b0: 1, b1: 2, b2: 1, a1: -0.1405360824207108, a2: 0.0049375976155402}, + {b0: 1, b1: -2, b2: 1, a1: -1.8849012174287920, a2: 0.8864214718161675}, + {b0: 1, b1: -2, b2: 1, a1: -1.9941388812663283, a2: 0.9941474694445309}, + }, offset: 0}, + CWeighting: {coeffs: []biquadCoeff{ + {b0: 1, b1: 2, b2: 1, a1: -0.1405360824207108, a2: 0.0049375976155402}, + {b0: 1, b1: -2, b2: 1, a1: -1.9941388812663283, a2: 0.9941474694445309}, + }, offset: 0}, + KWeighting: {coeffs: []biquadCoeff{ + {b0: 1.5308412300503476, b1: -2.6509799951547293, b2: 1.1690790799215869, a1: -1.6636551132560204, a2: 0.7125954280732254}, + {b0: 0.9995600645425144, b1: -1.9991201290850289, b2: 0.9995600645425144, a1: -1.9891696736297957, a2: 0.9891990357870394}, + }, offset: -0.691}, // offset is to make up for the fact that K-weighting has slightly above unity gain at 1 kHz + NoWeighting: {coeffs: []biquadCoeff{}, offset: 0}, +} + +// according to https://tech.ebu.ch/docs/tech/tech3341.pdf +// we have two sliding windows: momentary loudness = last 400 ms, short-term loudness = last 3 s +// display: +// +// momentary loudness = last analyzed 400 ms blcok +// short-term loudness = last analyzed 3 s block +// +// every 100 ms, we collect one data point of the momentary loudness (starting to play song again resets the data blocks) +// then: +// +// integrated loudness = the blocks are gated, and the average loudness of the gated blocks is calculated +// maximum momentary loudness = maximum of all the momentary blocks +// maximum short-term loudness = maximum of all the short-term blocks +func (d *loudnessDetector) update(chunk sointu.AudioBuffer) LoudnessResult { + l := max(len(chunk), MAX_INTEGRATED_DATA) + setSliceLength(&d.tmp, l) + setSliceLength(&d.tmp2, l) + setSliceLength(&d.tmpbool, l) + var total float32 + for chn := 0; chn < 2; chn++ { + // deinterleave the channels + for i := 0; i < len(chunk); i++ { + d.tmp[i] = chunk[i][chn] + } + // filter the signal with the weighting filter + for k := 0; k < len(d.weighting.coeffs); k++ { + d.states[chn][k].Filter(d.tmp[:len(chunk)], d.weighting.coeffs[k]) + } + // square the samples + res := vek32.Mul_Into(d.tmp2, d.tmp[:len(chunk)], d.tmp[:len(chunk)]) + // calculate the mean and add it to the total + total += vek32.Mean(res) + } + var ret [NumLoudnessTypes]Decibel + for i := range d.powers { + d.powers[i].WriteWrapSingle(total) // these are sliding windows of 4 and 30 power measurements (400 ms and 3 s aka momentary and short-term windows) + mean := vek32.Mean(d.powers[i].Buffer) + if len(d.averagedPowers[i]) < MAX_INTEGRATED_DATA { // we need to have some limit on how much data we keep + d.averagedPowers[i] = append(d.averagedPowers[i], mean) + } + if d.maxPowers[i] < mean { + d.maxPowers[i] = mean + } + ret[i+int(LoudnessMomentary)] = power2loudness(mean, d.weighting.offset) // we assume the LoudnessMomentary is followed by LoudnessShortTerm + ret[i+int(LoudnessMaxMomentary)] = power2loudness(d.maxPowers[i], d.weighting.offset) + } + if len(d.averagedPowers[0])%10 == 0 { // every 10 samples of 100 ms i.e. every 1 s, we recalculate the integrated power + absThreshold := loudness2power(-70, d.weighting.offset) // -70 dB is the first threshold + b := vek32.GtNumber_Into(d.tmpbool, d.averagedPowers[0], absThreshold) + m2 := vek32.Select_Into(d.tmp, d.averagedPowers[0], b) + if len(m2) > 0 { + relThreshold := vek32.Mean(m2) / 10 // the relative threshold is 10 dB below the mean of the values above the absolute threshold + b2 := vek32.GtNumber_Into(d.tmpbool, m2, relThreshold) + m3 := vek32.Select_Into(d.tmp2, m2, b2) + if len(m3) > 0 { + d.integratedPower = vek32.Mean(m3) + } + } + } + ret[LoudnessIntegrated] = power2loudness(d.integratedPower, d.weighting.offset) + return ret +} + +func (d *loudnessDetector) reset() { + for i := range d.powers { + d.powers[i].Cursor = 0 + l := len(d.powers[i].Buffer) + d.powers[i].Buffer = d.powers[i].Buffer[:0] + d.powers[i].Buffer = append(d.powers[i].Buffer, make([]float32, l)...) + d.averagedPowers[i] = d.averagedPowers[i][:0] + d.maxPowers[i] = 0 + } + d.integratedPower = 0 +} + +func power2loudness(power, offset float32) Decibel { + return Decibel(float32(10*math.Log10(float64(power))) + offset) +} + +func loudness2power(loudness Decibel, offset float32) float32 { + return (float32)(math.Pow(10, (float64(loudness)-float64(offset))/10)) +} + +func (state *biquadState) Filter(buffer []float32, coeff biquadCoeff) { + s := *state + for i := 0; i < len(buffer); i++ { + x := buffer[i] + y := coeff.b0*x + coeff.b1*s.x1 + coeff.b2*s.x2 - coeff.a1*s.y1 - coeff.a2*s.y2 + s.x2, s.x1 = s.x1, x + s.y2, s.y1 = s.y1, y + buffer[i] = y + } + *state = s +} + +func setSliceLength[T any](slice *[]T, length int) { + if len(*slice) < length { + *slice = append(*slice, make([]T, length-len(*slice))...) + } + *slice = (*slice)[:length] +} + +// ref: https://www.itu.int/dms_pubrec/itu-r/rec/bs/R-REC-BS.1770-5-202311-I!!PDF-E.pdf +var oversamplingCoeffs = [4][12]float32{ + {0.0017089843750, 0.0109863281250, -0.0196533203125, 0.0332031250000, -0.0594482421875, 0.1373291015625, 0.9721679687500, -0.1022949218750, 0.0476074218750, -0.0266113281250, 0.0148925781250, -0.0083007812500}, + {-0.0291748046875, 0.0292968750000, -0.0517578125000, 0.0891113281250, -0.1665039062500, 0.4650878906250, 0.7797851562500, -0.2003173828125, 0.1015625000000, -0.0582275390625, 0.0330810546875, -0.0189208984375}, + {-0.0189208984375, 0.0330810546875, -0.058227539062, 0.1015625000000, -0.200317382812, 0.7797851562500, 0.4650878906250, -0.166503906250, 0.0891113281250, -0.051757812500, 0.0292968750000, -0.0291748046875}, + {-0.0083007812500, 0.0148925781250, -0.0266113281250, 0.0476074218750, -0.1022949218750, 0.9721679687500, 0.1373291015625, -0.0594482421875, 0.0332031250000, -0.0196533203125, 0.0109863281250, 0.0017089843750}, +} + +// u[k] = x[k/4] if k%4 == 0, 0 otherwise +// y[k] = sum_{i=0}^{47} h[i] * u[k-i] +// h[i] = o[i%4][i/4] +// k = p*4+q, q=0..3 +// y[p*4+q] = sum_{j=0}^{11} sum_{i=0}^{3} h[j*4+i] * u[p*4+q-j*4-i] = ... +// (q-i)%4 == 0 ==> i = q +// ... = sum_{j=0}^{11} o[q][j] * x[p-j] +// y should be at least 4 times the length of x +func (s *oversamplerState) Oversample(x []float32, y []float32) []float32 { + if len(s.tmp) < len(x) { + s.tmp = append(s.tmp, make([]float32, len(x)-len(s.tmp))...) + } + if len(s.tmp2) < len(x) { + s.tmp2 = append(s.tmp2, make([]float32, len(x)-len(s.tmp2))...) + } + for q, coeffs := range oversamplingCoeffs { + // tmp2 will be conv(o[q],x) + r := vek32.Zeros_Into(s.tmp2, len(x)) + for j, c := range coeffs { + vek32.MulNumber_Into(s.tmp[:j], s.history[11-j:11], c) // convolution might pull values before x[0], so we need to use history for that + vek32.MulNumber_Into(s.tmp[j:], x[:len(x)-j], c) + vek32.Add_Inplace(r, s.tmp[:len(x)]) + } + // interleave the phases + for p, v := range r { + y[p*4+q] = v + } + } + z := min(len(x), 11) + copy(s.history[:11-z], s.history[z:11]) + copy(s.history[11-z:], x[len(x)-z:]) + return y[:len(x)*4] +} + +// we should perform the peak detection also momentary (last 400 ms), short term +// (last 3 s), and integrated (whole song) for display purposes, we can use +// always last arrived data for the integrated peak, we can use the maximum of +// all the peaks so far (there is no need show "maximum short term true peak" or +// "maximum momentary true peak" because they are same as the maximum for entire song) +// +// display: +// +// momentary true peak +// short-term true peak +// integrated true peak +func (d *peakDetector) update(buf sointu.AudioBuffer) (ret PeakResult) { + if len(d.tmp) < len(buf) { + d.tmp = append(d.tmp, make([]float32, len(buf)-len(d.tmp))...) + } + len4 := 4 * len(buf) + if len(d.tmp2) < len4 { + d.tmp2 = append(d.tmp2, make([]float32, len4-len(d.tmp2))...) + } + for chn := 0; chn < 2; chn++ { + // deinterleave the channels + for i := 0; i < len(buf); i++ { + d.tmp[i] = buf[i][chn] + } + // 4x oversample the signal + o := d.states[chn].Oversample(d.tmp[:len(buf)], d.tmp2) + // take absolute value of the oversampled signal + vek32.Abs_Inplace(o) + p := vek32.Max(o) + // find the maximum value in the window + for i := range d.windows { + d.windows[i][chn].WriteWrapSingle(p) + windowPeak := vek32.Max(d.windows[i][chn].Buffer) + ret[chn][i+int(PeakMomentary)] = Decibel(10 * math.Log10(float64(windowPeak))) + } + if d.maxPower[chn] < p { + d.maxPower[chn] = p + } + ret[int(PeakIntegrated)][chn] = Decibel(10 * math.Log10(float64(d.maxPower[chn]))) + } + return +} + +func (d *peakDetector) reset() { + for chn := 0; chn < 2; chn++ { + d.states[chn].history = [11]float32{} + for i := range d.windows[chn] { + d.windows[chn][i].Cursor = 0 + l := len(d.windows[chn][i].Buffer) + d.windows[chn][i].Buffer = d.windows[chn][i].Buffer[:0] + d.windows[chn][i].Buffer = append(d.windows[chn][i].Buffer, make([]float32, l)...) + } + d.maxPower[chn] = 0 + } +} diff --git a/tracker/gioui/oscilloscope.go b/tracker/gioui/oscilloscope.go new file mode 100644 index 0000000..35d3119 --- /dev/null +++ b/tracker/gioui/oscilloscope.go @@ -0,0 +1,183 @@ +package gioui + +import ( + "image" + "image/color" + "math" + + "gioui.org/io/event" + "gioui.org/io/pointer" + "gioui.org/layout" + "gioui.org/op/clip" + "gioui.org/op/paint" + "gioui.org/widget/material" + "github.com/vsariola/sointu/tracker" +) + +type ( + Oscilloscope struct { + onceBtn *BoolClickable + wrapBtn *BoolClickable + lengthInRowsNumber *NumberInput + triggerChannelNumber *NumberInput + xScale int + xOffset float32 + dragging bool + dragId pointer.ID + dragStartPx float32 + } + + OscilloscopeStyle struct { + Oscilloscope *Oscilloscope + Wave tracker.RingBuffer[[2]float32] + Colors [2]color.NRGBA + ClippedColor color.NRGBA + Theme *material.Theme + } +) + +func NewOscilloscope(model *tracker.Model) *Oscilloscope { + return &Oscilloscope{ + onceBtn: NewBoolClickable(model.SignalAnalyzer().Once().Bool()), + wrapBtn: NewBoolClickable(model.SignalAnalyzer().Wrap().Bool()), + lengthInRowsNumber: NewNumberInput(model.SignalAnalyzer().LengthInRows().Int()), + triggerChannelNumber: NewNumberInput(model.SignalAnalyzer().TriggerChannel().Int()), + } +} + +func LineOscilloscope(s *Oscilloscope, wave tracker.RingBuffer[[2]float32], th *material.Theme) *OscilloscopeStyle { + return &OscilloscopeStyle{Oscilloscope: s, Wave: wave, Colors: [2]color.NRGBA{primaryColor, secondaryColor}, Theme: th, ClippedColor: errorColor} +} + +func (s *OscilloscopeStyle) Layout(gtx C) D { + wrapBtnStyle := ToggleButton(gtx, s.Theme, s.Oscilloscope.wrapBtn, "Wrap") + onceBtnStyle := ToggleButton(gtx, s.Theme, s.Oscilloscope.onceBtn, "Once") + triggerChannelStyle := NumericUpDown(s.Theme, s.Oscilloscope.triggerChannelNumber, "Trigger channel") + lengthNumberStyle := NumericUpDown(s.Theme, s.Oscilloscope.lengthInRowsNumber, "Buffer length in rows") + return layout.Flex{Axis: layout.Vertical}.Layout(gtx, + layout.Flexed(1, func(gtx C) D { return s.layoutWave(gtx) }), + layout.Rigid(func(gtx C) D { + return layout.Flex{Axis: layout.Horizontal, Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(Label("TRG:", white, s.Theme.Shaper)), + layout.Rigid(triggerChannelStyle.Layout), + layout.Rigid(onceBtnStyle.Layout), + ) + }), + layout.Rigid(func(gtx C) D { + return layout.Flex{Axis: layout.Horizontal, Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(Label("BUF:", white, s.Theme.Shaper)), + layout.Rigid(lengthNumberStyle.Layout), + layout.Rigid(wrapBtnStyle.Layout), + ) + }), + ) +} + +func (s *OscilloscopeStyle) layoutWave(gtx C) D { + s.update(gtx) + if gtx.Constraints.Max.X == 0 || gtx.Constraints.Max.Y == 0 { + return D{} + } + defer clip.Rect(image.Rectangle{Max: gtx.Constraints.Max}).Push(gtx.Ops).Pop() + event.Op(gtx.Ops, s.Oscilloscope) + paint.ColorOp{Color: disabledTextColor}.Add(gtx.Ops) + cursorX := int(s.sampleToPx(gtx, float32(s.Wave.Cursor))) + stack := clip.Rect{Min: image.Pt(cursorX, 0), Max: image.Pt(cursorX+1, gtx.Constraints.Max.Y)}.Push(gtx.Ops) + paint.PaintOp{}.Add(gtx.Ops) + stack.Pop() + for chn := 0; chn < 2; chn++ { + paint.ColorOp{Color: s.Colors[chn]}.Add(gtx.Ops) + clippedColorSet := false + yprev := int((s.Wave.Buffer[0][chn] + 1) / 2 * float32(gtx.Constraints.Max.Y)) + for px := 0; px < gtx.Constraints.Max.X; px++ { + x := int(s.pxToSample(gtx, float32(px))) + if x < 0 || x >= len(s.Wave.Buffer) { + continue + } + y := int((s.Wave.Buffer[x][chn] + 1) / 2 * float32(gtx.Constraints.Max.Y)) + if y < 0 { + y = 0 + } else if y >= gtx.Constraints.Max.Y { + y = gtx.Constraints.Max.Y - 1 + } + y1, y2 := yprev, y + if y < yprev { + y1, y2 = y, yprev-1 + } else if y > yprev { + y1++ + } + clipped := false + if y1 == y2 && y1 == 0 { + clipped = true + } + if y1 == y2 && y1 == gtx.Constraints.Max.Y-1 { + clipped = true + } + if clippedColorSet != clipped { + if clipped { + paint.ColorOp{Color: s.ClippedColor}.Add(gtx.Ops) + } else { + paint.ColorOp{Color: s.Colors[chn]}.Add(gtx.Ops) + } + clippedColorSet = clipped + } + stack := clip.Rect{Min: image.Pt(px, y1), Max: image.Pt(px+1, y2+1)}.Push(gtx.Ops) + paint.PaintOp{}.Add(gtx.Ops) + stack.Pop() + yprev = y + } + } + return D{Size: image.Pt(gtx.Constraints.Max.X, gtx.Constraints.Max.Y)} +} + +func (o *OscilloscopeStyle) update(gtx C) { + for { + ev, ok := gtx.Event(pointer.Filter{ + Target: o.Oscilloscope, + Kinds: pointer.Scroll | pointer.Press | pointer.Drag | pointer.Release | pointer.Cancel, + ScrollY: pointer.ScrollRange{Min: -1e6, Max: 1e6}, + }) + if !ok { + break + } + if e, ok := ev.(pointer.Event); ok { + switch e.Kind { + case pointer.Scroll: + s1 := o.pxToSample(gtx, e.Position.X) + o.Oscilloscope.xScale += min(max(-1, int(e.Scroll.Y)), 1) + s2 := o.pxToSample(gtx, e.Position.X) + o.Oscilloscope.xOffset -= s1 - s2 + case pointer.Press: + if e.Buttons&pointer.ButtonSecondary != 0 { + o.Oscilloscope.xOffset = 0 + o.Oscilloscope.xScale = 0 + } + if e.Buttons&pointer.ButtonPrimary != 0 { + o.Oscilloscope.dragging = true + o.Oscilloscope.dragId = e.PointerID + o.Oscilloscope.dragStartPx = e.Position.X + } + case pointer.Drag: + if e.Buttons&pointer.ButtonPrimary != 0 && o.Oscilloscope.dragging && e.PointerID == o.Oscilloscope.dragId { + delta := o.pxToSample(gtx, e.Position.X) - o.pxToSample(gtx, o.Oscilloscope.dragStartPx) + o.Oscilloscope.xOffset += delta + o.Oscilloscope.dragStartPx = e.Position.X + } + case pointer.Release | pointer.Cancel: + o.Oscilloscope.dragging = false + } + } + } +} + +func (o *OscilloscopeStyle) scaleFactor() float32 { + return float32(math.Pow(1.1, float64(o.Oscilloscope.xScale))) +} + +func (s *OscilloscopeStyle) pxToSample(gtx C, px float32) float32 { + return px*s.scaleFactor()*float32(len(s.Wave.Buffer))/float32(gtx.Constraints.Max.X) - s.Oscilloscope.xOffset +} + +func (s *OscilloscopeStyle) sampleToPx(gtx C, sample float32) float32 { + return (sample + s.Oscilloscope.xOffset) * float32(gtx.Constraints.Max.X) / float32(len(s.Wave.Buffer)) / s.scaleFactor() +} diff --git a/tracker/gioui/songpanel.go b/tracker/gioui/songpanel.go index 7a0ef88..1d42adf 100644 --- a/tracker/gioui/songpanel.go +++ b/tracker/gioui/songpanel.go @@ -29,6 +29,8 @@ type SongPanel struct { PanicBtn *BoolClickable LoopBtn *BoolClickable + Scope *Oscilloscope + // File menu items fileMenuItems []MenuItem NewSong tracker.Action @@ -68,6 +70,7 @@ func NewSongPanel(model *tracker.Model) *SongPanel { FollowBtn: NewBoolClickable(model.Follow().Bool()), PlayingBtn: NewBoolClickable(model.Playing().Bool()), RewindBtn: NewActionClickable(model.PlaySongStart()), + Scope: NewOscilloscope(model), } ret.fileMenuItems = []MenuItem{ {IconBytes: icons.ContentClear, Text: "New Song", ShortcutText: keyActionMap["NewSong"], Doer: model.NewSong()}, @@ -120,9 +123,16 @@ func (t *SongPanel) layoutMenuBar(gtx C, tr *Tracker) D { gtx.Constraints.Max.Y = gtx.Dp(unit.Dp(36)) gtx.Constraints.Min.Y = gtx.Dp(unit.Dp(36)) + panicBtnStyle := ToggleIcon(gtx, tr.Theme, t.PanicBtn, icons.AlertErrorOutline, icons.AlertError, t.panicHint, t.panicHint) + if t.PanicBtn.Bool.Value() { + panicBtnStyle.IconButtonStyle.Color = errorColor + } menuLayouts := []layout.FlexChild{ layout.Rigid(tr.layoutMenu(gtx, "File", &t.MenuBar[0], &t.Menus[0], unit.Dp(200), t.fileMenuItems...)), layout.Rigid(tr.layoutMenu(gtx, "Edit", &t.MenuBar[1], &t.Menus[1], unit.Dp(200), t.editMenuItems...)), + layout.Flexed(1, func(gtx C) D { + return layout.E.Layout(gtx, panicBtnStyle.Layout) + }), } if len(t.midiMenuItems) > 0 { menuLayouts = append( @@ -138,13 +148,14 @@ func (t *SongPanel) layoutSongOptions(gtx C, tr *Tracker) D { in := layout.UniformInset(unit.Dp(1)) - panicBtnStyle := ToggleButton(gtx, tr.Theme, t.PanicBtn, t.panicHint) rewindBtnStyle := ActionIcon(gtx, tr.Theme, t.RewindBtn, icons.AVFastRewind, t.rewindHint) playBtnStyle := ToggleIcon(gtx, tr.Theme, t.PlayingBtn, icons.AVPlayArrow, icons.AVStop, t.playHint, t.stopHint) recordBtnStyle := ToggleIcon(gtx, tr.Theme, t.RecordBtn, icons.AVFiberManualRecord, icons.AVFiberSmartRecord, t.recordHint, t.stopRecordHint) noteTrackBtnStyle := ToggleIcon(gtx, tr.Theme, t.FollowBtn, icons.ActionSpeakerNotesOff, icons.ActionSpeakerNotes, t.followOffHint, t.followOnHint) loopBtnStyle := ToggleIcon(gtx, tr.Theme, t.LoopBtn, icons.NavigationArrowForward, icons.AVLoop, t.loopOffHint, t.loopOnHint) + scopeStyle := LineOscilloscope(t.Scope, tr.SignalAnalyzer().Waveform(), tr.Theme) + return layout.Flex{Axis: layout.Vertical}.Layout(gtx, layout.Rigid(func(gtx C) D { return layout.Flex{Axis: layout.Horizontal}.Layout(gtx, @@ -205,7 +216,7 @@ func (t *SongPanel) layoutSongOptions(gtx C, tr *Tracker) D { }), ) }), - layout.Rigid(VuMeter{AverageVolume: tr.Model.AverageVolume(), PeakVolume: tr.Model.PeakVolume(), Range: 100}.Layout), + layout.Rigid(VuMeter{Loudness: tr.Model.DetectorResult().Loudness[tracker.LoudnessShortTerm], Peak: tr.Model.DetectorResult().Peaks[tracker.PeakMomentary], Range: 100}.Layout), layout.Rigid(func(gtx C) D { return layout.Flex{Axis: layout.Horizontal, Alignment: layout.Middle}.Layout(gtx, layout.Rigid(rewindBtnStyle.Layout), @@ -215,8 +226,7 @@ func (t *SongPanel) layoutSongOptions(gtx C, tr *Tracker) D { layout.Rigid(loopBtnStyle.Layout), ) }), - layout.Rigid(panicBtnStyle.Layout), - layout.Flexed(1, func(gtx C) D { return layout.Dimensions{Size: gtx.Constraints.Min} }), + layout.Flexed(1, scopeStyle.Layout), layout.Rigid(func(gtx C) D { labelStyle := LabelStyle{Text: version.VersionOrHash, FontSize: unit.Sp(12), Color: mediumEmphasisTextColor, Shaper: tr.Theme.Shaper} return labelStyle.Layout(gtx) diff --git a/tracker/gioui/tracker.go b/tracker/gioui/tracker.go index 115e14b..b80dab8 100644 --- a/tracker/gioui/tracker.go +++ b/tracker/gioui/tracker.go @@ -116,8 +116,8 @@ func (t *Tracker) Main() { var ops op.Ops for { select { - case e := <-t.PlayerMessages: - t.ProcessPlayerMessage(e) + case e := <-t.Broker().ToModel: + t.ProcessMsg(e) w.Invalidate() case e := <-events: switch e := e.(type) { @@ -166,6 +166,7 @@ func (t *Tracker) Main() { w.Perform(system.ActionClose) t.SaveRecovery() t.quitWG.Done() + t.Broker().Close() } func eventLoop(w *app.Window, events chan<- event.Event, acks <-chan struct{}) { diff --git a/tracker/gioui/vumeter.go b/tracker/gioui/vumeter.go index aeb4d87..7a47183 100644 --- a/tracker/gioui/vumeter.go +++ b/tracker/gioui/vumeter.go @@ -11,9 +11,9 @@ import ( ) type VuMeter struct { - AverageVolume tracker.Volume - PeakVolume tracker.Volume - Range float32 + Loudness tracker.Decibel + Peak [2]tracker.Decibel + Range float32 } func (v VuMeter) Layout(gtx C) D { @@ -21,7 +21,7 @@ func (v VuMeter) Layout(gtx C) D { gtx.Constraints.Max.Y = gtx.Dp(unit.Dp(12)) height := gtx.Dp(unit.Dp(6)) for j := 0; j < 2; j++ { - value := float32(v.AverageVolume[j]) + v.Range + value := float32(v.Loudness) + v.Range if value > 0 { x := int(value/v.Range*float32(gtx.Constraints.Max.X) + 0.5) if x > gtx.Constraints.Max.X { @@ -29,7 +29,7 @@ func (v VuMeter) Layout(gtx C) D { } paint.FillShape(gtx.Ops, mediumEmphasisTextColor, clip.Rect(image.Rect(0, 0, x, height)).Op()) } - valueMax := float32(v.PeakVolume[j]) + v.Range + valueMax := float32(v.Peak[j]) + v.Range if valueMax > 0 { color := white if valueMax >= v.Range { diff --git a/tracker/model.go b/tracker/model.go index 3612b04..3d00b28 100644 --- a/tracker/model.go +++ b/tracker/model.go @@ -70,15 +70,15 @@ type ( cachePatternUseCount [][]int voiceLevels [vm.MAX_VOICES]float32 - avgVolume Volume - peakVolume Volume + + signalAnalyzer *ScopeModel + detectorResult DetectorResult alerts []Alert dialog Dialog synther sointu.Synther // the synther used to create new synths - PlayerMessages chan PlayerMsg - ModelMessages chan<- interface{} + broker *Broker MIDI MIDIContext trackMidiIn bool @@ -171,8 +171,6 @@ const ( const maxUndo = 64 -func (m *Model) AverageVolume() Volume { return m.avgVolume } -func (m *Model) PeakVolume() Volume { return m.peakVolume } func (m *Model) PlayPosition() sointu.SongPos { return m.playPosition } func (m *Model) Loop() Loop { return m.loop } func (m *Model) PlaySongRow() int { return m.d.Song.Score.SongRow(m.playPosition) } @@ -180,16 +178,15 @@ func (m *Model) ChangedSinceSave() bool { return m.d.ChangedSinceSave } func (m *Model) Dialog() Dialog { return m.dialog } func (m *Model) Quitted() bool { return m.quitted } +func (m *Model) DetectorResult() DetectorResult { return m.detectorResult } + // NewModelPlayer creates a new model and a player that communicates with it -func NewModelPlayer(synther sointu.Synther, midiContext MIDIContext, recoveryFilePath string) (*Model, *Player) { +func NewModelPlayer(broker *Broker, synther sointu.Synther, midiContext MIDIContext, recoveryFilePath string) (*Model, *Player) { m := new(Model) m.synther = synther m.MIDI = midiContext m.trackMidiIn = midiContext.HasDeviceOpen() - modelMessages := make(chan interface{}, 1024) - playerMessages := make(chan PlayerMsg, 1024) - m.ModelMessages = modelMessages - m.PlayerMessages = playerMessages + m.broker = broker m.d.Octave = 4 m.linkInstrTrack = true m.d.RecoveryFilePath = recoveryFilePath @@ -199,14 +196,12 @@ func NewModelPlayer(synther sointu.Synther, midiContext MIDIContext, recoveryFil json.Unmarshal(bytes2, &m.d) } } + m.signalAnalyzer = NewScopeModel(broker, m.d.Song.BPM) p := &Player{ - playerMsgs: playerMessages, - modelMsgs: modelMessages, - synther: synther, - song: m.d.Song.Copy(), - loop: m.loop, - avgVolumeMeter: VolumeAnalyzer{Attack: 0.3, Release: 0.3, Min: -100, Max: 20}, - peakVolumeMeter: VolumeAnalyzer{Attack: 1e-4, Release: 1, Min: -100, Max: 20}, + broker: broker, + synther: synther, + song: m.d.Song.Copy(), + loop: m.loop, } p.compileOrUpdateSynth() return m, p @@ -262,6 +257,7 @@ func (m *Model) change(kind string, t ChangeType, severity ChangeSeverity) func( } if m.changeType&BPMChange != 0 { m.send(BPMMsg{m.d.Song.BPM}) + m.signalAnalyzer.SetBpm(m.d.Song.BPM) } if m.changeType&RowsPerBeatChange != 0 { m.send(RowsPerBeatMsg{m.d.Song.RowsPerBeat}) @@ -344,17 +340,26 @@ func (m *Model) UnmarshalRecovery(bytes []byte) { m.updatePatternUseCount() } -func (m *Model) ProcessPlayerMessage(msg PlayerMsg) { - m.playPosition = msg.SongPosition - m.voiceLevels = msg.VoiceLevels - m.avgVolume = msg.AverageVolume - m.peakVolume = msg.PeakVolume - if m.playing && m.follow { - m.d.Cursor.SongPos = msg.SongPosition - m.d.Cursor2.SongPos = msg.SongPosition +func (m *Model) ProcessMsg(msg MsgToModel) { + if msg.HasPanicPosLevels { + m.playPosition = msg.SongPosition + m.voiceLevels = msg.VoiceLevels + if m.playing && m.follow { + m.d.Cursor.SongPos = msg.SongPosition + m.d.Cursor2.SongPos = msg.SongPosition + } + m.panic = msg.Panic } - m.panic = msg.Panic - switch e := msg.Inner.(type) { + if msg.HasDetectorResult { + m.detectorResult = msg.DetectorResult + } + if msg.TriggerChannel > 0 { + m.signalAnalyzer.Trigger(msg.TriggerChannel) + } + if msg.Reset { + m.signalAnalyzer.Reset() + } + switch e := msg.Data.(type) { case func(): e() case Recording: @@ -373,10 +378,15 @@ func (m *Model) ProcessPlayerMessage(msg PlayerMsg) { m.Alerts().AddAlert(e) case IsPlayingMsg: m.playing = e.bool + case *sointu.AudioBuffer: + m.signalAnalyzer.ProcessAudioBuffer(e) default: } } +func (m *Model) SignalAnalyzer() *ScopeModel { return m.signalAnalyzer } +func (m *Model) Broker() *Broker { return m.broker } + func (m *Model) TrackNoteOn(track int, note byte) (id NoteID) { id = NoteID{IsInstr: false, Track: track, Note: note, model: m} m.send(NoteOnMsg{id}) @@ -424,7 +434,7 @@ func (m *Model) resetSong() { // send sends a message to the player func (m *Model) send(message interface{}) { - m.ModelMessages <- message + m.broker.ToPlayer <- message } func (m *Model) maxID() int { diff --git a/tracker/model_test.go b/tracker/model_test.go index 886efdf..6e1d737 100644 --- a/tracker/model_test.go +++ b/tracker/model_test.go @@ -23,6 +23,8 @@ func (NullContext) BPM() (bpm float64, ok bool) { func (NullContext) InputDevices(yield func(tracker.MIDIDevice) bool) {} +func (NullContext) HasDeviceOpen() bool { return false } + func (NullContext) Close() {} type modelFuzzState struct { @@ -260,7 +262,9 @@ func FuzzModel(f *testing.F) { f.Fuzz(func(t *testing.T, slice []byte) { reader := bytes.NewReader(slice) synther := vm.GoSynther{} - model, player := tracker.NewModelPlayer(synther, NullContext{}, "") + broker := tracker.NewBroker() + defer broker.Close() + model, player := tracker.NewModelPlayer(broker, synther, NullContext{}, "") buf := make([][2]float32, 2048) closeChan := make(chan struct{}) go func() { diff --git a/tracker/player.go b/tracker/player.go index 3534769..0abc996 100644 --- a/tracker/player.go +++ b/tracker/player.go @@ -15,23 +15,20 @@ type ( // model via the playerMessages channel. The model sends messages to the // player via the modelMessages channel. Player struct { - synth sointu.Synth // the synth used to render audio - song sointu.Song // the song being played - playing bool // is the player playing the score or not - rowtime int // how many samples have been played in the current row - songPos sointu.SongPos // the current position in the score - avgVolumeMeter VolumeAnalyzer // the volume analyzer used to calculate the average volume - peakVolumeMeter VolumeAnalyzer // the volume analyzer used to calculate the peak volume - voiceLevels [vm.MAX_VOICES]float32 // a level that can be used to visualize the volume of each voice - voices [vm.MAX_VOICES]voice - loop Loop + synth sointu.Synth // the synth used to render audio + song sointu.Song // the song being played + playing bool // is the player playing the score or not + rowtime int // how many samples have been played in the current row + songPos sointu.SongPos // the current position in the score + voiceLevels [vm.MAX_VOICES]float32 // a level that can be used to visualize the volume of each voice + voices [vm.MAX_VOICES]voice + loop Loop recState recState // is the recording off; are we waiting for a note; or are we recording recording Recording // the recorded MIDI events and BPM - synther sointu.Synther // the synther used to create new synths - playerMsgs chan<- PlayerMsg - modelMsgs <-chan interface{} + synther sointu.Synther // the synther used to create new synths + broker *Broker // the broker used to communicate with different parts of the tracker } // PlayerProcessContext is the context given to the player when processing @@ -55,20 +52,6 @@ type ( Channel int Note byte } - - // PlayerMsg is a message sent from the player to the model. The Inner - // field can contain any message. Panic, AverageVolume, PeakVolume, SongRow - // and VoiceStates transmitted frequently, with every message, so they are - // treated specially, to avoid boxing. All the rest messages can be boxed to - // Inner interface{} - PlayerMsg struct { - Panic bool - AverageVolume Volume - PeakVolume Volume - SongPosition sointu.SongPos - VoiceLevels [vm.MAX_VOICES]float32 - Inner interface{} - } ) type ( @@ -105,8 +88,6 @@ func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext p.recording.TotalFrames += len(buffer) } - oldBuffer := buffer - for i := 0; i < numRenderTries; i++ { for midiOk && frame >= midi.Frame { if p.recState == recStateWaitingForNote { @@ -162,6 +143,14 @@ func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext p.synth = nil p.send(Alert{Message: fmt.Sprintf("synth.Render: %s", err.Error()), Priority: Error, Name: "PlayerCrash"}) } + + bufPtr := p.broker.GetAudioBuffer() // borrow a buffer from the broker + *bufPtr = append(*bufPtr, buffer[:rendered]...) + if len(*bufPtr) == 0 || !trySend(p.broker.ToModel, MsgToModel{Data: bufPtr}) { + // if the buffer is empty or sending the rendered waveform to Model + // failed, return the buffer to the broker + p.broker.PutAudioBuffer(bufPtr) + } buffer = buffer[rendered:] frame += rendered p.rowtime += timeAdvanced @@ -178,18 +167,6 @@ func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext } // when the buffer is full, return if len(buffer) == 0 { - err := p.avgVolumeMeter.Update(oldBuffer) - err2 := p.peakVolumeMeter.Update(oldBuffer) - if err != nil { - p.synth = nil - p.SendAlert("PlayerVolume", err.Error(), Warning) - return - } - if err2 != nil { - p.synth = nil - p.SendAlert("PlayerVolume", err2.Error(), Warning) - return - } p.send(nil) return } @@ -239,7 +216,7 @@ func (p *Player) processMessages(context PlayerProcessContext, uiProcessor Event loop: for { // process new message select { - case msg := <-p.modelMsgs: + case msg := <-p.broker.ToPlayer: switch m := msg.(type) { case PanicMsg: if m.bool { @@ -263,6 +240,8 @@ loop: for i := range p.song.Score.Tracks { p.releaseTrack(i) } + } else { + trySend(p.broker.ToModel, MsgToModel{Reset: true}) } case BPMMsg: p.song.BPM = m.int @@ -281,6 +260,7 @@ loop: p.releaseTrack(i) } } + trySend(p.broker.ToModel, MsgToModel{Reset: true}) case NoteOnMsg: if m.IsInstr { p.triggerInstrument(m.Instr, m.Note) @@ -358,10 +338,7 @@ func (p *Player) compileOrUpdateSynth() { // all sends from player are always non-blocking, to ensure that the player thread cannot end up in a dead-lock func (p *Player) send(message interface{}) { - select { - case p.playerMsgs <- PlayerMsg{Panic: p.synth == nil, AverageVolume: p.avgVolumeMeter.Level, PeakVolume: p.peakVolumeMeter.Level, SongPosition: p.songPos, VoiceLevels: p.voiceLevels, Inner: message}: - default: - } + trySend(p.broker.ToModel, MsgToModel{HasPanicPosLevels: true, Panic: p.synth == nil, SongPosition: p.songPos, VoiceLevels: p.voiceLevels, Data: message}) } func (p *Player) triggerInstrument(instrument int, note byte) { @@ -417,6 +394,7 @@ func (p *Player) trigger(voiceStart, voiceEnd int, note byte, ID int) { p.voices[oldestVoice] = voice{noteID: ID, sustain: true, samplesSinceEvent: 0} p.voiceLevels[oldestVoice] = 1.0 p.synth.Trigger(oldestVoice, note) + trySend(p.broker.ToModel, MsgToModel{TriggerChannel: instrIndex + 1}) } func (p *Player) release(ID int) { diff --git a/tracker/scopemodel.go b/tracker/scopemodel.go new file mode 100644 index 0000000..eb39607 --- /dev/null +++ b/tracker/scopemodel.go @@ -0,0 +1,141 @@ +package tracker + +import ( + "github.com/vsariola/sointu" + "github.com/vsariola/sointu/vm" +) + +type ( + ScopeModel struct { + waveForm RingBuffer[[2]float32] + once bool + triggered bool + wrap bool + triggerChannel int + lengthInRows int + bpm int + + broker *Broker + } + + RingBuffer[T any] struct { + Buffer []T + Cursor int + } + + SignalOnce ScopeModel + SignalWrap ScopeModel + SignalLengthInRows ScopeModel + TriggerChannel ScopeModel +) + +func (r *RingBuffer[T]) WriteWrap(values []T) { + r.Cursor = (r.Cursor + len(values)) % len(r.Buffer) + a := min(len(values), r.Cursor) // how many values to copy before the cursor + b := min(len(values)-a, len(r.Buffer)-r.Cursor) // how many values to copy to the end of the buffer + copy(r.Buffer[r.Cursor-a:r.Cursor], values[len(values)-a:]) + copy(r.Buffer[len(r.Buffer)-b:], values[len(values)-a-b:]) +} + +func (r *RingBuffer[T]) WriteWrapSingle(value T) { + r.Cursor = (r.Cursor + 1) % len(r.Buffer) + r.Buffer[r.Cursor] = value +} + +func (r *RingBuffer[T]) WriteOnce(values []T) { + if r.Cursor < len(r.Buffer) { + r.Cursor += copy(r.Buffer[r.Cursor:], values) + } +} + +func (r *RingBuffer[T]) WriteOnceSingle(value T) { + if r.Cursor < len(r.Buffer) { + r.Buffer[r.Cursor] = value + r.Cursor++ + } +} + +func NewScopeModel(broker *Broker, bpm int) *ScopeModel { + s := &ScopeModel{ + broker: broker, + bpm: bpm, + lengthInRows: 4, + } + s.updateBufferLength() + return s +} + +func (s *ScopeModel) Waveform() RingBuffer[[2]float32] { return s.waveForm } + +func (s *ScopeModel) Once() *SignalOnce { return (*SignalOnce)(s) } +func (s *ScopeModel) Wrap() *SignalWrap { return (*SignalWrap)(s) } +func (s *ScopeModel) LengthInRows() *SignalLengthInRows { return (*SignalLengthInRows)(s) } +func (s *ScopeModel) TriggerChannel() *TriggerChannel { return (*TriggerChannel)(s) } + +func (m *SignalOnce) Bool() Bool { return Bool{m} } +func (m *SignalOnce) Value() bool { return m.once } +func (m *SignalOnce) setValue(val bool) { m.once = val } +func (m *SignalOnce) Enabled() bool { return true } + +func (m *SignalWrap) Bool() Bool { return Bool{m} } +func (m *SignalWrap) Value() bool { return m.wrap } +func (m *SignalWrap) setValue(val bool) { m.wrap = val } +func (m *SignalWrap) Enabled() bool { return true } + +func (m *SignalLengthInRows) Int() Int { return Int{m} } +func (m *SignalLengthInRows) Value() int { return m.lengthInRows } +func (m *SignalLengthInRows) setValue(val int) { + m.lengthInRows = val + (*ScopeModel)(m).updateBufferLength() +} +func (m *SignalLengthInRows) Enabled() bool { return true } +func (m *SignalLengthInRows) Range() intRange { return intRange{1, 999} } +func (m *SignalLengthInRows) change(string) func() { return func() {} } + +func (m *TriggerChannel) Int() Int { return Int{m} } +func (m *TriggerChannel) Value() int { return m.triggerChannel } +func (m *TriggerChannel) setValue(val int) { m.triggerChannel = val } +func (m *TriggerChannel) Enabled() bool { return true } +func (m *TriggerChannel) Range() intRange { return intRange{0, vm.MAX_VOICES} } +func (m *TriggerChannel) change(string) func() { return func() {} } + +func (s *ScopeModel) ProcessAudioBuffer(bufPtr *sointu.AudioBuffer) { + if s.wrap { + s.waveForm.WriteWrap(*bufPtr) + } else { + s.waveForm.WriteOnce(*bufPtr) + } + // chain the messages: when we have a new audio buffer, try passing it on to the detector + if !trySend(s.broker.ToDetector, MsgToDetector{Data: bufPtr}) { + s.broker.PutAudioBuffer(bufPtr) + } +} + +// Note: channel 1 is the first channel +func (s *ScopeModel) Trigger(channel int) { + if s.triggerChannel > 0 && channel == s.triggerChannel && !(s.once && s.triggered) { + s.waveForm.Cursor = 0 + s.triggered = true + } +} + +func (s *ScopeModel) Reset() { + s.waveForm.Cursor = 0 + s.triggered = false + l := len(s.waveForm.Buffer) + s.waveForm.Buffer = s.waveForm.Buffer[:0] + s.waveForm.Buffer = append(s.waveForm.Buffer, make([][2]float32, l)...) + trySend(s.broker.ToDetector, MsgToDetector{Reset: true}) // chain the messages: when the signal analyzer is reset, also reset the detector +} + +func (s *ScopeModel) SetBpm(bpm int) { + s.bpm = bpm + s.updateBufferLength() +} + +func (s *ScopeModel) updateBufferLength() { + if s.bpm == 0 || s.lengthInRows == 0 { + return + } + setSliceLength(&s.waveForm.Buffer, 44100*60*s.lengthInRows/s.bpm) +} diff --git a/tracker/volume.go b/tracker/volume.go deleted file mode 100644 index fb079a3..0000000 --- a/tracker/volume.go +++ /dev/null @@ -1,68 +0,0 @@ -package tracker - -import ( - "errors" - "math" - - "github.com/vsariola/sointu" -) - -type ( - Volume [2]float64 - - // VolumeAnalyzer measures the volume in an AudioBuffer, in decibels relative to - // full scale (0 dB = signal level of +-1) - VolumeAnalyzer struct { - Level Volume // current volume level of left and right channels - Attack float64 // attack time constant in seconds - Release float64 // release time constant in seconds - Min float64 // minimum volume in decibels - Max float64 // maximum volume in decibels - } -) - -var nanError = errors.New("NaN detected in master output") - -// Update updates the Level field, by analyzing the given buffer. -// -// Internally, it first converts the signal to decibels (0 dB = +-1). Then, the -// average volume level is computed by smoothing the decibel values with a -// exponentially decaying average, with a time constant Attack (in seconds) if -// the decibel value is greater than current level and time constant Decay (in -// seconds) if the decibel value is less than current level. -// -// Typical time constants for average level detection would be 0.3 seconds for -// both attack and release. For peak level detection, attack could be 1.5e-3 and -// release 1.5 (seconds) -// -// MinVolume and MaxVolume are hard limits in decibels to prevent negative -// infinities for volumes -func (v *VolumeAnalyzer) Update(buffer sointu.AudioBuffer) (err error) { - // from https://en.wikipedia.org/wiki/Exponential_smoothing - alphaAttack := 1 - math.Exp(-1.0/(v.Attack*44100)) - alphaRelease := 1 - math.Exp(-1.0/(v.Release*44100)) - for j := 0; j < 2; j++ { - for i := 0; i < len(buffer); i++ { - sample2 := float64(buffer[i][j] * buffer[i][j]) - if math.IsNaN(sample2) { - if err == nil { - err = nanError - } - continue - } - dB := 10 * math.Log10(sample2) - if dB < v.Min || math.IsNaN(dB) { - dB = v.Min - } - if dB > v.Max { - dB = v.Max - } - a := alphaAttack - if dB < v.Level[j] { - a = alphaRelease - } - v.Level[j] += (dB - v.Level[j]) * a - } - } - return err -}