From 9b9dc3548f3da1fe2ffb8998926c1c6d44cfc2f9 Mon Sep 17 00:00:00 2001 From: "5684185+vsariola@users.noreply.github.com" <5684185+vsariola@users.noreply.github.com> Date: Fri, 31 Oct 2025 19:40:02 +0200 Subject: [PATCH] feat: add multithreaded rendering to the tracker side The compiled player does not support multithreading, but with this, users can already start composing songs with slightly less powerful machines, even when targeting high-end machines. Related to #199 --- CHANGELOG.md | 7 + audio.go | 23 +++ cmd/synthers.go | 1 + cmd/synthers_native.go | 6 +- patch.go | 35 +++- song.go | 5 +- tracker/bool.go | 80 +++++++++ tracker/gioui/instrument_properties.go | 22 ++- tracker/gioui/song_panel.go | 43 ++++- tracker/model.go | 4 +- tracker/player.go | 43 +++-- vm/compiler/bridge/native_synth.go | 33 +++- vm/compiler/bridge/native_synth_test.go | 5 + vm/go_synth.go | 36 ++-- vm/go_synth_test.go | 2 + vm/multithread_synth.go | 217 ++++++++++++++++++++++++ 16 files changed, 500 insertions(+), 62 deletions(-) create mode 100644 vm/multithread_synth.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f4e881c..bb04feb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +- Multithreaded synths: the user can split the patch up to four threads. + Selecting the thread can be done on the instrument properties pane. + Multithreading works only on the multithreaded synths, selectable from the CPU + panel. Currently the multithreaded rendering has not yet been implemented in + the compiled player and the thread information is disregarded while compiling + the song. ([#199][i199]) - Preset explorer, whichs allows 1) searching the presets by name; 2) filtering them by category (directory); 3) filtering them by being builtin vs. user; 4) filtering them if they need gm.dls (for Linux/Mac users, who don't have @@ -382,6 +388,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). [i186]: https://github.com/vsariola/sointu/issues/186 [i192]: https://github.com/vsariola/sointu/issues/192 [i196]: https://github.com/vsariola/sointu/issues/196 +[i199]: https://github.com/vsariola/sointu/issues/199 [i200]: https://github.com/vsariola/sointu/issues/200 [i210]: https://github.com/vsariola/sointu/issues/210 [i211]: https://github.com/vsariola/sointu/issues/211 diff --git a/audio.go b/audio.go index 4c7f347..cc2f12c 100644 --- a/audio.go +++ b/audio.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "math" + "time" ) type ( @@ -62,6 +63,13 @@ type ( // Release releases the currently playing note for a given voice. Called // between synth.Renders. Release(voice int) + + // Close disposes the synth, freeing any resources. No other functions should be called after Close. + Close() + + // Populates the given array with the current CPU load of each thread, + // returning the number of threads / elements populated + CPULoad([]CPULoad) int } // Synther compiles a given Patch into a Synth, throwing errors if the @@ -69,7 +77,10 @@ type ( Synther interface { Name() string // Name of the synther, e.g. "Go" or "Native" Synth(patch Patch, bpm int) (Synth, error) + SupportsMultithreading() bool } + + CPULoad float32 ) // Play plays the Song by first compiling the patch with the given Synther, @@ -83,6 +94,7 @@ func Play(synther Synther, song Song, progress func(float32)) (AudioBuffer, erro if err != nil { return nil, fmt.Errorf("sointu.Play failed: %v", err) } + defer synth.Close() curVoices := make([]int, len(song.Score.Tracks)) for i := range curVoices { curVoices[i] = song.Score.FirstVoiceForTrack(i) @@ -204,6 +216,17 @@ func (buffer AudioBuffer) Raw(pcm16 bool) ([]byte, error) { return buf.Bytes(), nil } +func (p *CPULoad) Update(duration time.Duration, frames int64) { + if frames <= 0 { + return // no frames rendered, so cannot compute CPU load + } + realtime := float64(duration) / 1e9 + songtime := float64(frames) / 44100 + newload := realtime / songtime + alpha := math.Exp(-songtime) // smoothing factor, time constant of 1 second + *p = CPULoad(float64(*p)*alpha + newload*(1-alpha)) +} + func (data AudioBuffer) rawToBuffer(pcm16 bool, buf *bytes.Buffer) error { var err error if pcm16 { diff --git a/cmd/synthers.go b/cmd/synthers.go index 1022c1e..7da1d1c 100644 --- a/cmd/synthers.go +++ b/cmd/synthers.go @@ -7,4 +7,5 @@ import ( var Synthers = []sointu.Synther{ vm.GoSynther{}, + vm.MakeMultithreadSynther(vm.GoSynther{}), } diff --git a/cmd/synthers_native.go b/cmd/synthers_native.go index 144ff87..3ef7009 100644 --- a/cmd/synthers_native.go +++ b/cmd/synthers_native.go @@ -2,8 +2,12 @@ package cmd -import "github.com/vsariola/sointu/vm/compiler/bridge" +import ( + "github.com/vsariola/sointu/vm" + "github.com/vsariola/sointu/vm/compiler/bridge" +) func init() { Synthers = append(Synthers, bridge.NativeSynther{}) + Synthers = append(Synthers, vm.MakeMultithreadSynther(bridge.NativeSynther{})) } diff --git a/patch.go b/patch.go index a7bca4b..b5afee7 100644 --- a/patch.go +++ b/patch.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "math" + "math/bits" "sort" "strconv" @@ -19,8 +20,12 @@ type ( Name string `yaml:",omitempty"` Comment string `yaml:",omitempty"` NumVoices int - Units []Unit Mute bool `yaml:",omitempty"` // Mute is only used in the tracker for soloing/muting instruments; the compiled player ignores this field + // ThreadMaskM1 is a bit mask of which threads are used, minus 1. Minus + // 1 is done so that the default value 0 means bit mask 0b0001 i.e. only + // thread 1 is rendering the instrument. + ThreadMaskM1 int `yaml:",omitempty"` + Units []Unit } // Unit is e.g. a filter, oscillator, envelope and its parameters @@ -347,13 +352,14 @@ func init() { // Copy makes a deep copy of a unit. func (u *Unit) Copy() Unit { - parameters := make(map[string]int) + ret := *u + ret.Parameters = make(map[string]int, len(u.Parameters)) for k, v := range u.Parameters { - parameters[k] = v + ret.Parameters[k] = v } - varArgs := make([]int, len(u.VarArgs)) - copy(varArgs, u.VarArgs) - return Unit{Type: u.Type, Parameters: parameters, VarArgs: varArgs, ID: u.ID, Disabled: u.Disabled, Comment: u.Comment} + ret.VarArgs = make([]int, len(u.VarArgs)) + copy(ret.VarArgs, u.VarArgs) + return ret } var stackUseSource = [2]StackUse{ @@ -473,11 +479,12 @@ func (u *Unit) StackNeed() int { // Copy makes a deep copy of an Instrument func (instr *Instrument) Copy() Instrument { - units := make([]Unit, len(instr.Units)) + ret := *instr + ret.Units = make([]Unit, len(instr.Units)) for i, u := range instr.Units { - units[i] = u.Copy() + ret.Units[i] = u.Copy() } - return Instrument{Name: instr.Name, Comment: instr.Comment, NumVoices: instr.NumVoices, Units: units, Mute: instr.Mute} + return ret } // Implement the counter interface @@ -536,6 +543,16 @@ func (p Patch) NumSyncs() int { return total } +func (p Patch) NumThreads() int { + numThreads := 1 + for _, instr := range p { + if l := bits.Len((uint)(instr.ThreadMaskM1 + 1)); l > numThreads { + numThreads = l + } + } + return numThreads +} + // FirstVoiceForInstrument returns the index of the first voice of given // instrument. For example, if the Patch has three instruments (0, 1 and 2), // with 1, 3, 2 voices, respectively, then FirstVoiceForInstrument(0) returns 0, diff --git a/song.go b/song.go index 2bf81da..06df5f4 100644 --- a/song.go +++ b/song.go @@ -293,7 +293,10 @@ func (l Score) LengthInRows() int { // Copy makes a deep copy of a Score. func (s *Song) Copy() Song { - return Song{BPM: s.BPM, RowsPerBeat: s.RowsPerBeat, Score: s.Score.Copy(), Patch: s.Patch.Copy()} + ret := *s + ret.Score = s.Score.Copy() + ret.Patch = s.Patch.Copy() + return ret } // Assuming 44100 Hz playback speed, return the number of samples of each row of diff --git a/tracker/bool.go b/tracker/bool.go index 46b91b8..353a1bf 100644 --- a/tracker/bool.go +++ b/tracker/bool.go @@ -1,5 +1,9 @@ package tracker +import ( + "fmt" +) + type ( Bool struct { value BoolValue @@ -29,6 +33,10 @@ type ( InstrEditor Model InstrPresets Model InstrComment Model + Thread1 Model + Thread2 Model + Thread3 Model + Thread4 Model ) func MakeBool(valueEnabler interface { @@ -66,6 +74,78 @@ func (v Bool) Enabled() bool { return v.enabler.Enabled() } +// Thread methods + +func (m *Model) getThreadsBit(bit int) bool { + if m.d.InstrIndex < 0 || m.d.InstrIndex >= len(m.d.Song.Patch) { + return false + } + mask := m.d.Song.Patch[m.d.InstrIndex].ThreadMaskM1 + 1 + return mask&(1<= len(m.d.Song.Patch) { + return + } + defer (*Model)(m).change("ThreadBitMask", PatchChange, MinorChange)() + mask := m.d.Song.Patch[m.d.InstrIndex].ThreadMaskM1 + 1 + if value { + mask |= (1 << bit) + } else { + mask &^= (1 << bit) + } + m.d.Song.Patch[m.d.InstrIndex].ThreadMaskM1 = max(mask-1, 0) // -1 would have all threads disabled, so make that 0 i.e. use at least thread 1 + m.warnAboutCrossThreadSends() + m.warnNoMultithreadSupport() +} + +func (m *Model) warnAboutCrossThreadSends() { + for i, instr := range m.d.Song.Patch { + for _, unit := range instr.Units { + if unit.Type == "send" { + targetID, ok := unit.Parameters["target"] + if !ok { + continue + } + it, _, err := m.d.Song.Patch.FindUnit(targetID) + if err != nil { + continue + } + if instr.ThreadMaskM1 != m.d.Song.Patch[it].ThreadMaskM1 { + m.Alerts().AddNamed("CrossThreadSend", fmt.Sprintf("Instrument %d '%s' has a send to instrument %d '%s' but they are not on the same threads, which may cause issues", i+1, instr.Name, it+1, m.d.Song.Patch[it].Name), Warning) + return + } + } + } + } +} + +func (m *Model) warnNoMultithreadSupport() { + for _, instr := range m.d.Song.Patch { + if instr.ThreadMaskM1 > 0 && !m.synthers[m.syntherIndex].SupportsMultithreading() { + m.Alerts().AddNamed("NoMultithreadSupport", "The current synth does not support multithreading and the patch was configured to use more than one thread", Warning) + return + } + } +} + +func (m *Model) Thread1() Bool { return MakeEnabledBool((*Thread1)(m)) } +func (m *Thread1) Value() bool { return (*Model)(m).getThreadsBit(0) } +func (m *Thread1) SetValue(val bool) { (*Model)(m).setThreadsBit(0, val) } + +func (m *Model) Thread2() Bool { return MakeEnabledBool((*Thread2)(m)) } +func (m *Thread2) Value() bool { return (*Model)(m).getThreadsBit(1) } +func (m *Thread2) SetValue(val bool) { (*Model)(m).setThreadsBit(1, val) } + +func (m *Model) Thread3() Bool { return MakeEnabledBool((*Thread3)(m)) } +func (m *Thread3) Value() bool { return (*Model)(m).getThreadsBit(2) } +func (m *Thread3) SetValue(val bool) { (*Model)(m).setThreadsBit(2, val) } + +func (m *Model) Thread4() Bool { return MakeEnabledBool((*Thread4)(m)) } +func (m *Thread4) Value() bool { return (*Model)(m).getThreadsBit(3) } +func (m *Thread4) SetValue(val bool) { (*Model)(m).setThreadsBit(3, val) } + // Panic methods func (m *Model) Panic() Bool { return MakeEnabledBool((*Panic)(m)) } diff --git a/tracker/gioui/instrument_properties.go b/tracker/gioui/instrument_properties.go index d8d964c..a544033 100644 --- a/tracker/gioui/instrument_properties.go +++ b/tracker/gioui/instrument_properties.go @@ -19,6 +19,7 @@ type ( list *layout.List soloBtn *Clickable muteBtn *Clickable + threadBtns [4]*Clickable soloHint string unsoloHint string muteHint string @@ -38,6 +39,7 @@ func NewInstrumentProperties() *InstrumentProperties { muteBtn: new(Clickable), voices: NewNumericUpDownState(), splitInstrumentBtn: new(Clickable), + threadBtns: [4]*Clickable{new(Clickable), new(Clickable), new(Clickable), new(Clickable)}, } ret.soloHint = makeHint("Solo", " (%s)", "SoloToggle") ret.unsoloHint = makeHint("Unsolo", " (%s)", "SoloToggle") @@ -66,7 +68,21 @@ func (ip *InstrumentProperties) layout(gtx C) D { ) } - return ip.list.Layout(gtx, 9, func(gtx C, index int) D { + thread1btn := ToggleIconBtn(tr.Thread1(), tr.Theme, ip.threadBtns[0], icons.ImageCropSquare, icons.ImageFilter1, "Do not render instrument on thread 1", "Render instrument on thread 1") + thread2btn := ToggleIconBtn(tr.Thread2(), tr.Theme, ip.threadBtns[1], icons.ImageCropSquare, icons.ImageFilter2, "Do not render instrument on thread 2", "Render instrument on thread 2") + thread3btn := ToggleIconBtn(tr.Thread3(), tr.Theme, ip.threadBtns[2], icons.ImageCropSquare, icons.ImageFilter3, "Do not render instrument on thread 3", "Render instrument on thread 3") + thread4btn := ToggleIconBtn(tr.Thread4(), tr.Theme, ip.threadBtns[3], icons.ImageCropSquare, icons.ImageFilter4, "Do not render instrument on thread 4", "Render instrument on thread 4") + + threadbtnline := func(gtx C) D { + return layout.Flex{Axis: layout.Horizontal, Alignment: layout.Middle}.Layout(gtx, + layout.Rigid(thread1btn.Layout), + layout.Rigid(thread2btn.Layout), + layout.Rigid(thread3btn.Layout), + layout.Rigid(thread4btn.Layout), + ) + } + + return ip.list.Layout(gtx, 11, func(gtx C, index int) D { switch index { case 0: return layoutInstrumentPropertyLine(gtx, "Name", func(gtx C) D { @@ -81,6 +97,8 @@ func (ip *InstrumentProperties) layout(gtx C) D { soloBtn := ToggleIconBtn(tr.Solo(), tr.Theme, ip.soloBtn, icons.ToggleCheckBoxOutlineBlank, icons.ToggleCheckBox, ip.soloHint, ip.unsoloHint) return layoutInstrumentPropertyLine(gtx, "Solo", soloBtn.Layout) case 8: + return layoutInstrumentPropertyLine(gtx, "Thread", threadbtnline) + case 10: return layout.UniformInset(unit.Dp(6)).Layout(gtx, func(gtx C) D { return ip.commentEditor.Layout(gtx, tr.InstrumentComment(), tr.Theme, &tr.Theme.InstrumentEditor.InstrumentComment, "Comment") }) @@ -94,7 +112,7 @@ func (ip *InstrumentProperties) layout(gtx C) D { func layoutInstrumentPropertyLine(gtx C, text string, content layout.Widget) D { tr := TrackerFromContext(gtx) - gtx.Constraints.Max.X = min(gtx.Dp(unit.Dp(200)), gtx.Constraints.Max.X) + gtx.Constraints.Max.X = min(gtx.Dp(300), gtx.Constraints.Max.X) label := Label(tr.Theme, &tr.Theme.InstrumentEditor.Properties.Label, text) return layout.Flex{Axis: layout.Horizontal, Alignment: layout.Middle}.Layout(gtx, layout.Rigid(layout.Spacer{Width: 6, Height: 36}.Layout), diff --git a/tracker/gioui/song_panel.go b/tracker/gioui/song_panel.go index dd71651..807c8e2 100644 --- a/tracker/gioui/song_panel.go +++ b/tracker/gioui/song_panel.go @@ -4,15 +4,19 @@ import ( "fmt" "image" "image/color" + "slices" "strconv" + "strings" "gioui.org/gesture" "gioui.org/layout" "gioui.org/op/clip" "gioui.org/op/paint" "gioui.org/unit" + "github.com/vsariola/sointu" "github.com/vsariola/sointu/tracker" "github.com/vsariola/sointu/version" + "github.com/vsariola/sointu/vm" "golang.org/x/exp/shiny/materialdesign/icons" ) @@ -110,10 +114,37 @@ func (t *SongPanel) layoutSongOptions(gtx C) D { } oversamplingBtn := Btn(tr.Theme, &tr.Theme.Button.Text, t.OversamplingBtn, oversamplingTxt, "") - cpuload := tr.Model.CPULoad() - cpuLabel := Label(tr.Theme, &tr.Theme.SongPanel.RowValue, fmt.Sprintf("%.0f %%", cpuload*100)) - if cpuload >= 1 { - cpuLabel.Color = tr.Theme.SongPanel.ErrorColor + cpuSmallLabel := func(gtx C) D { + var a [vm.MAX_THREADS]sointu.CPULoad + c := tr.Model.CPULoad(a[:]) + load := slices.Max(a[:c]) + cpuLabel := Label(tr.Theme, &tr.Theme.SongPanel.RowValue, fmt.Sprintf("%d%%", int(load*100+0.5))) + if load >= 1 { + cpuLabel.Color = tr.Theme.SongPanel.ErrorColor + } + return cpuLabel.Layout(gtx) + } + + cpuEnlargedWidget := func(gtx C) D { + var sb strings.Builder + var a [vm.MAX_THREADS]sointu.CPULoad + c := tr.Model.CPULoad(a[:]) + high := false + for i := range c { + if i > 0 { + fmt.Fprint(&sb, ", ") + } + cpuLoad := a[i] + fmt.Fprintf(&sb, "%d%%", int(cpuLoad*100+0.5)) + if cpuLoad >= 1 { + high = true + } + } + cpuLabel := Label(tr.Theme, &tr.Theme.SongPanel.RowValue, sb.String()) + if high { + cpuLabel.Color = tr.Theme.SongPanel.ErrorColor + } + return cpuLabel.Layout(gtx) } synthBtn := Btn(tr.Theme, &tr.Theme.Button.Text, t.SynthBtn, tr.Model.SyntherName(), "") @@ -150,10 +181,10 @@ func (t *SongPanel) layoutSongOptions(gtx C) D { }) }), layout.Rigid(func(gtx C) D { - return t.CPUExpander.Layout(gtx, tr.Theme, "CPU", cpuLabel.Layout, + return t.CPUExpander.Layout(gtx, tr.Theme, "CPU", cpuSmallLabel, func(gtx C) D { return layout.Flex{Axis: layout.Vertical, Alignment: layout.End}.Layout(gtx, - layout.Rigid(func(gtx C) D { return layoutSongOptionRow(gtx, tr.Theme, "Load", cpuLabel.Layout) }), + layout.Rigid(func(gtx C) D { return layoutSongOptionRow(gtx, tr.Theme, "Load", cpuEnlargedWidget) }), layout.Rigid(func(gtx C) D { return layoutSongOptionRow(gtx, tr.Theme, "Synth", synthBtn.Layout) }), ) }, diff --git a/tracker/model.go b/tracker/model.go index 90264fb..6d37ca4 100644 --- a/tracker/model.go +++ b/tracker/model.go @@ -397,7 +397,9 @@ func (m *Model) ProcessMsg(msg MsgToModel) { } } -func (m *Model) CPULoad() float64 { return m.playerStatus.CPULoad } +func (m *Model) CPULoad(buf []sointu.CPULoad) int { + return copy(buf, m.playerStatus.CPULoad[:m.playerStatus.NumThreads]) +} func (m *Model) SignalAnalyzer() *ScopeModel { return m.signalAnalyzer } func (m *Model) Broker() *Broker { return m.broker } diff --git a/tracker/player.go b/tracker/player.go index 247e62e..54eb944 100644 --- a/tracker/player.go +++ b/tracker/player.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "slices" - "time" "github.com/vsariola/sointu" "github.com/vsariola/sointu/vm" @@ -42,7 +41,8 @@ type ( PlayerStatus struct { SongPos sointu.SongPos // the current position in the score VoiceLevels [vm.MAX_VOICES]float32 // a level that can be used to visualize the volume of each voice - CPULoad float64 // current CPU load of the player, used to adjust the render rate + NumThreads int + CPULoad [vm.MAX_THREADS]sointu.CPULoad // current CPU load of the player, used to adjust the render rate } // PlayerProcessContext is the context given to the player when processing @@ -97,9 +97,6 @@ func NewPlayer(broker *Broker, synther sointu.Synther) *Player { // buffer. It is used to trigger and release notes during processing. The // context is also used to get the current BPM from the host. func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext) { - startTime := time.Now() - startFrame := p.frame - p.processMessages(context) p.events.adjustTimes(p.frameDeltas, p.frame, p.frame+int64(len(buffer))) @@ -127,12 +124,12 @@ func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext if p.synth != nil { rendered, timeAdvanced, err = p.synth.Render(buffer[:framesUntilEvent], timeUntilRowAdvance) if err != nil { - p.synth = nil + p.destroySynth() p.send(Alert{Message: fmt.Sprintf("synth.Render: %s", err.Error()), Priority: Error, Name: "PlayerCrash", Duration: defaultAlertDuration}) } // for performance, we don't check for NaN of every sample, because typically NaNs propagate if rendered > 0 && (isNaN(buffer[0][0]) || isNaN(buffer[0][1]) || isInf(buffer[0][0]) || isInf(buffer[0][1])) { - p.synth = nil + p.destroySynth() p.send(Alert{Message: "Inf or NaN detected in synth output", Priority: Error, Name: "PlayerCrash", Duration: defaultAlertDuration}) } } else { @@ -164,17 +161,26 @@ func (p *Player) Process(buffer sointu.AudioBuffer, context PlayerProcessContext } // when the buffer is full, return if len(buffer) == 0 { - p.updateCPULoad(time.Since(startTime), p.frame-startFrame) + if p.synth != nil { + p.status.NumThreads = p.synth.CPULoad(p.status.CPULoad[:]) + } p.send(nil) return } } // we were not able to fill the buffer with NUM_RENDER_TRIES attempts, destroy synth and throw an error - p.synth = nil + p.destroySynth() p.events = p.events[:0] // clear events, so we don't try to process them again p.SendAlert("PlayerCrash", fmt.Sprintf("synth did not fill the audio buffer even with %d render calls", numRenderTries), Error) } +func (p *Player) destroySynth() { + if p.synth != nil { + p.synth.Close() + p.synth = nil + } +} + func (p *Player) advanceRow() { if p.song.Score.Length == 0 || p.song.Score.RowsPerPattern == 0 { return @@ -227,7 +233,7 @@ loop: switch m := msg.(type) { case PanicMsg: if m.bool { - p.synth = nil + p.destroySynth() } else { p.compileOrUpdateSynth() } @@ -283,7 +289,7 @@ loop: } case sointu.Synther: p.synther = m - p.synth = nil + p.destroySynth() p.compileOrUpdateSynth() default: // ignore unknown messages @@ -355,7 +361,7 @@ func (p *Player) compileOrUpdateSynth() { if p.synth != nil { err := p.synth.Update(p.song.Patch, p.song.BPM) if err != nil { - p.synth = nil + p.destroySynth() p.SendAlert("PlayerCrash", fmt.Sprintf("synth.Update: %v", err), Error) return } @@ -363,7 +369,7 @@ func (p *Player) compileOrUpdateSynth() { var err error p.synth, err = p.synther.Synth(p.song.Patch, p.song.BPM) if err != nil { - p.synth = nil + p.destroySynth() p.SendAlert("PlayerCrash", fmt.Sprintf("synther.Synth: %v", err), Error) return } @@ -441,14 +447,3 @@ func (p *Player) processNoteEvent(ev NoteEvent) { p.synth.Trigger(oldestVoice, ev.Note) TrySend(p.broker.ToModel, MsgToModel{TriggerChannel: instrIndex + 1}) } - -func (p *Player) updateCPULoad(duration time.Duration, frames int64) { - if frames <= 0 { - return // no frames rendered, so cannot compute CPU load - } - realtime := float64(duration) / 1e9 - songtime := float64(frames) / 44100 - newload := realtime / songtime - alpha := math.Exp(-songtime) // smoothing factor, time constant of 1 second - p.status.CPULoad = float64(p.status.CPULoad)*alpha + newload*(1-alpha) -} diff --git a/vm/compiler/bridge/native_synth.go b/vm/compiler/bridge/native_synth.go index 4c3c087..9ffcd93 100644 --- a/vm/compiler/bridge/native_synth.go +++ b/vm/compiler/bridge/native_synth.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/vsariola/sointu" "github.com/vsariola/sointu/vm" @@ -16,9 +17,13 @@ import ( type NativeSynther struct { } -type NativeSynth C.Synth +type NativeSynth struct { + csynth C.Synth + cpuLoad sointu.CPULoad +} -func (s NativeSynther) Name() string { return "Native" } +func (s NativeSynther) Name() string { return "Native" } +func (s NativeSynther) SupportsMultithreading() bool { return false } func (s NativeSynther) Synth(patch sointu.Patch, bpm int) (sointu.Synth, error) { synth, err := Synth(patch, bpm) @@ -45,7 +50,7 @@ func Synth(patch sointu.Patch, bpm int) (*NativeSynth, error) { s.Opcodes[0] = 0 s.NumVoices = 1 s.Polyphony = 0 - return (*NativeSynth)(s), nil + return &NativeSynth{csynth: *s}, nil } for i, v := range comPatch.Opcodes { s.Opcodes[i] = (C.uchar)(v) @@ -64,7 +69,17 @@ func Synth(patch sointu.Patch, bpm int) (*NativeSynth, error) { s.NumVoices = C.uint(comPatch.NumVoices) s.Polyphony = C.uint(comPatch.PolyphonyBitmask) s.RandSeed = 1 - return (*NativeSynth)(s), nil + return &NativeSynth{csynth: *s}, nil +} + +func (s *NativeSynth) Close() {} + +func (s *NativeSynth) CPULoad(loads []sointu.CPULoad) int { + if len(loads) < 1 { + return 0 + } + loads[0] = s.cpuLoad + return 1 } // Render renders until the buffer is full or the modulated time is reached, whichever @@ -89,12 +104,14 @@ func Synth(patch sointu.Patch, bpm int) (*NativeSynth, error) { // exit condition would fire when the time is already past maxtime. // Under no conditions, nsamples >= len(buffer)/2 i.e. guaranteed to never overwrite the buffer. func (bridgesynth *NativeSynth) Render(buffer sointu.AudioBuffer, maxtime int) (int, int, error) { - synth := (*C.Synth)(bridgesynth) + synth := &bridgesynth.csynth // TODO: syncBuffer is not getting passed to cgo; do we want to even try to support the syncing with the native bridge if len(buffer)%1 == 1 { return -1, -1, errors.New("RenderTime writes stereo signals, so buffer should have even length") } samples := C.int(len(buffer)) + startTime := time.Now() + defer func() { bridgesynth.cpuLoad.Update(time.Since(startTime), int64(samples)) }() time := C.int(maxtime) errcode := int(C.su_render(synth, (*C.float)(&buffer[0][0]), &samples, &time)) if errcode > 0 { @@ -105,7 +122,7 @@ func (bridgesynth *NativeSynth) Render(buffer sointu.AudioBuffer, maxtime int) ( // Trigger is part of C.Synths' implementation of sointu.Synth interface func (bridgesynth *NativeSynth) Trigger(voice int, note byte) { - s := (*C.Synth)(bridgesynth) + s := &bridgesynth.csynth if voice < 0 || voice >= len(s.SynthWrk.Voices) { return } @@ -116,7 +133,7 @@ func (bridgesynth *NativeSynth) Trigger(voice int, note byte) { // Release is part of C.Synths' implementation of sointu.Synth interface func (bridgesynth *NativeSynth) Release(voice int) { - s := (*C.Synth)(bridgesynth) + s := &bridgesynth.csynth if voice < 0 || voice >= len(s.SynthWrk.Voices) { return } @@ -125,7 +142,7 @@ func (bridgesynth *NativeSynth) Release(voice int) { // Update func (bridgesynth *NativeSynth) Update(patch sointu.Patch, bpm int) error { - s := (*C.Synth)(bridgesynth) + s := &bridgesynth.csynth if n := patch.NumDelayLines(); n > 128 { return fmt.Errorf("native bridge has currently a hard limit of 128 delaylines; patch uses %v", n) } diff --git a/vm/compiler/bridge/native_synth_test.go b/vm/compiler/bridge/native_synth_test.go index c1398c5..382c201 100644 --- a/vm/compiler/bridge/native_synth_test.go +++ b/vm/compiler/bridge/native_synth_test.go @@ -86,6 +86,7 @@ func TestRenderSamples(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() synth.Trigger(0, 64) buffer := make(sointu.AudioBuffer, su_max_samples) err = buffer[:len(buffer)/2].Fill(synth) @@ -162,6 +163,7 @@ func TestStackUnderflow(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { @@ -178,6 +180,7 @@ func TestStackBalancing(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { @@ -211,6 +214,7 @@ func TestStackOverflow(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { @@ -228,6 +232,7 @@ func TestDivideByZero(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { diff --git a/vm/go_synth.go b/vm/go_synth.go index 9d0dbfc..89a3e08 100644 --- a/vm/go_synth.go +++ b/vm/go_synth.go @@ -7,6 +7,7 @@ import ( "math" "os" "path/filepath" + "time" "github.com/vsariola/sointu" ) @@ -27,6 +28,7 @@ type ( stack []float32 state synthState delaylines []delayline + cpuLoad sointu.CPULoad } // GoSynther is a Synther implementation that can converts patches into @@ -93,7 +95,8 @@ success: f.Read(su_sample_table[:]) } -func (s GoSynther) Name() string { return "Go" } +func (s GoSynther) Name() string { return "Go" } +func (s GoSynther) SupportsMultithreading() bool { return false } func (s GoSynther) Synth(patch sointu.Patch, bpm int) (sointu.Synth, error) { bytecode, err := NewBytecode(patch, AllFeatures{}, bpm) @@ -115,6 +118,16 @@ func (s *GoSynth) Release(voiceIndex int) { s.state.voices[voiceIndex].sustain = false } +func (s *GoSynth) Close() {} + +func (s *GoSynth) CPULoad(loads []sointu.CPULoad) int { + if len(loads) < 1 { + return 0 + } + loads[0] = s.cpuLoad + return 1 +} + func (s *GoSynth) Update(patch sointu.Patch, bpm int) error { bytecode, err := NewBytecode(patch, AllFeatures{}, bpm) if err != nil { @@ -143,7 +156,10 @@ func (s *GoSynth) Update(patch sointu.Patch, bpm int) error { return nil } -func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, time int, renderError error) { +func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, renderTime int, renderError error) { + startTime := time.Now() + defer func() { s.cpuLoad.Update(time.Since(startTime), int64(samples)) }() + defer func() { if err := recover(); err != nil { renderError = fmt.Errorf("render panicced: %v", err) @@ -153,7 +169,7 @@ func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, t stack := s.stack[:] stack = append(stack, []float32{0, 0, 0, 0}...) synth := &s.state - for time < maxtime && len(buffer) > 0 { + for renderTime < maxtime && len(buffer) > 0 { opcodesInstr := s.bytecode.Opcodes operandsInstr := s.bytecode.Operands opcodes, operands := opcodesInstr, operandsInstr @@ -182,7 +198,7 @@ func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, t } tcount := transformCounts[opNoStereo-1] if len(operands) < tcount { - return samples, time, errors.New("operand stream ended prematurely") + return samples, renderTime, errors.New("operand stream ended prematurely") } voice := &voices[0] unit := &units[0] @@ -289,7 +305,7 @@ func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, t r := unit.state[0] + float32(math.Exp2(float64(stack[l-1]*2.206896551724138))-1) w := int(r+1.5) - 1 unit.state[0] = r - float32(w) - time += w + renderTime += w stack = stack[:l-1] case opIn: var channel byte @@ -581,26 +597,26 @@ func (s *GoSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, t case opSync: break default: - return samples, time, errors.New("invalid / unimplemented opcode") + return samples, renderTime, errors.New("invalid / unimplemented opcode") } units = units[1:] } if len(stack) < 4 { - return samples, time, errors.New("stack underflow") + return samples, renderTime, errors.New("stack underflow") } if len(stack) > 4 { - return samples, time, errors.New("stack not empty") + return samples, renderTime, errors.New("stack not empty") } buffer[0][0], buffer[0][1] = synth.outputs[0], synth.outputs[1] synth.outputs[0] = 0 synth.outputs[1] = 0 buffer = buffer[1:] samples++ - time++ + renderTime++ s.state.globalTime++ } s.stack = stack[:0] - return samples, time, nil + return samples, renderTime, nil } func (s *synthState) rand() float32 { diff --git a/vm/go_synth_test.go b/vm/go_synth_test.go index 49ddd55..d03b204 100644 --- a/vm/go_synth_test.go +++ b/vm/go_synth_test.go @@ -197,6 +197,7 @@ func TestStackUnderflow(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { @@ -213,6 +214,7 @@ func TestStackBalancing(t *testing.T) { if err != nil { t.Fatalf("bridge compile error: %v", err) } + defer synth.Close() buffer := make(sointu.AudioBuffer, 1) err = buffer.Fill(synth) if err == nil { diff --git a/vm/multithread_synth.go b/vm/multithread_synth.go new file mode 100644 index 0000000..026084d --- /dev/null +++ b/vm/multithread_synth.go @@ -0,0 +1,217 @@ +package vm + +import ( + "math" + "math/bits" + "runtime" + "sync" + + "github.com/vsariola/sointu" +) + +type ( + MultithreadSynth struct { + voiceMapping voiceMapping + synths []sointu.Synth + commands chan<- multithreadSynthCommand // maxtime + results <-chan multithreadSynthResult // rendered buffer + pool sync.Pool + synther sointu.Synther + } + + MultithreadSynther struct { + synther sointu.Synther + name string + } + + voiceMapping [MAX_THREADS][MAX_VOICES]int + + multithreadSynthCommand struct { + thread int + samples int + time int + } + + multithreadSynthResult struct { + buffer *sointu.AudioBuffer + samples int + time int + renderError error + } +) + +const MAX_THREADS = 4 + +func MakeMultithreadSynther(synther sointu.Synther) MultithreadSynther { + return MultithreadSynther{synther: synther, name: "Multithread " + synther.Name()} +} + +func (s MultithreadSynther) Name() string { return s.name } +func (s MultithreadSynther) SupportsMultithreading() bool { return true } + +func (s MultithreadSynther) Synth(patch sointu.Patch, bpm int) (sointu.Synth, error) { + patches, voiceMapping := splitPatchByCores(patch) + synths := make([]sointu.Synth, 0, len(patches)) + for _, p := range patches { + synth, err := s.synther.Synth(p, bpm) + if err != nil { + return nil, err + } + synths = append(synths, synth) + } + ret := &MultithreadSynth{ + synths: synths, + voiceMapping: voiceMapping, + pool: sync.Pool{New: func() any { ret := make(sointu.AudioBuffer, 0, 8096); return &ret }}, + } + ret.startProcesses() + ret.synther = s.synther + return ret, nil +} + +func (s *MultithreadSynth) Update(patch sointu.Patch, bpm int) error { + patches, voiceMapping := splitPatchByCores(patch) + if s.voiceMapping != voiceMapping { + s.voiceMapping = voiceMapping + s.closeSynths() + } + for i, p := range patches { + if len(s.synths) <= i { + synth, err := s.synther.Synth(p, bpm) + if err != nil { + s.closeSynths() + return err + } + s.synths = append(s.synths, synth) + } else { + if err := s.synths[i].Update(p, bpm); err != nil { + s.closeSynths() + return err + } + } + } + return nil +} + +func (s *MultithreadSynth) startProcesses() { + maxProcs := runtime.GOMAXPROCS(0) + cmdChan := make(chan multithreadSynthCommand, maxProcs) + s.commands = cmdChan + resultsChan := make(chan multithreadSynthResult, maxProcs) + s.results = resultsChan + for i := 0; i < maxProcs; i++ { + go func(commandCh <-chan multithreadSynthCommand, resultCh chan<- multithreadSynthResult) { + for cmd := range commandCh { + buffer := s.pool.Get().(*sointu.AudioBuffer) + *buffer = append(*buffer, make(sointu.AudioBuffer, cmd.samples)...) + samples, time, renderError := s.synths[cmd.thread].Render(*buffer, cmd.time) + resultCh <- multithreadSynthResult{buffer: buffer, samples: samples, time: time, renderError: renderError} + } + }(cmdChan, resultsChan) + } +} + +func (s *MultithreadSynth) Close() { + close(s.commands) + s.closeSynths() +} + +func (s *MultithreadSynth) closeSynths() { + for _, synth := range s.synths { + synth.Close() + } + s.synths = s.synths[:0] +} + +func (s *MultithreadSynth) Trigger(voiceIndex int, note byte) { + for i, synth := range s.synths { + if ind := s.voiceMapping[i][voiceIndex]; ind >= 0 { + synth.Trigger(ind, note) + } + } +} + +func (s *MultithreadSynth) Release(voiceIndex int) { + for i, synth := range s.synths { + if ind := s.voiceMapping[i][voiceIndex]; ind >= 0 { + synth.Release(ind) + } + } +} + +func (s *MultithreadSynth) CPULoad(loads []sointu.CPULoad) (elems int) { + for _, synth := range s.synths { + n := synth.CPULoad(loads) + elems += n + loads = loads[n:] + if len(loads) <= 0 { + return + } + } + return +} + +func (s *MultithreadSynth) Render(buffer sointu.AudioBuffer, maxtime int) (samples int, time int, renderError error) { + count := len(s.synths) + for i := 0; i < count; i++ { + s.commands <- multithreadSynthCommand{thread: i, samples: len(buffer), time: maxtime} + } + clear(buffer) + samples = math.MaxInt + time = math.MaxInt + for i := 0; i < count; i++ { + // We mix the results as they come, but the order doesn't matter. This + // leads to slight indeterminism in the results, because the order of + // floating point additions can change the least significant bits. + result := <-s.results + if result.renderError != nil && renderError == nil { + renderError = result.renderError + } + samples = min(samples, result.samples) + time = min(time, result.time) + for j := 0; j < samples; j++ { + buffer[j][0] += (*result.buffer)[j][0] + buffer[j][1] += (*result.buffer)[j][1] + } + *result.buffer = (*result.buffer)[:0] + s.pool.Put(result.buffer) + } + return +} + +func splitPatchByCores(patch sointu.Patch) ([]sointu.Patch, voiceMapping) { + cores := 1 + for _, instr := range patch { + cores = max(bits.Len((uint)(instr.ThreadMaskM1+1)), cores) + } + cores = min(cores, MAX_THREADS) + ret := make([]sointu.Patch, cores) + for c := 0; c < cores; c++ { + ret[c] = make(sointu.Patch, 0, len(patch)) + } + var voicemapping [MAX_THREADS][MAX_VOICES]int + for c := 0; c < MAX_THREADS; c++ { + for j := 0; j < MAX_VOICES; j++ { + voicemapping[c][j] = -1 + } + } + for c := range cores { + coreVoice := 0 + curVoice := 0 + for _, instr := range patch { + mask := instr.ThreadMaskM1 + 1 + if mask&(1<= MAX_VOICES { + break + } + voicemapping[c][curVoice+j] = coreVoice + j + } + coreVoice += instr.NumVoices + } + curVoice += instr.NumVoices + } + } + return ret, voicemapping +}