to mumble jitter buffer and silence

Makefile

@@ -1,4 +1,4 @@
-GOFILES=main.go mumble.go discord.go bridge.go config.go mumble-handlers.go discord-handlers.go tickerct.go
+GOFILES=main.go mumble.go discord.go bridge.go config.go mumble-handlers.go discord-handlers.go sleepct.go
 
 mumble-discord-bridge: $(GOFILES)
 	goreleaser build --skip-validate --rm-dist

README.md

@@ -47,7 +47,9 @@ Usage of ./mumble-discord-bridge:
   -nice
      NICE, whether the bridge should automatically try to 'nice' itself, (default false)
   -to-discord-buffer int
-        TO_DISCORD_BUFFER, Delay buffer from Mumble to Discord to absorb timing issues related to network and hardware quality. (Increments of 10ms) (default 50)
+     TO_DISCORD_BUFFER, Jitter buffer from Mumble to Discord to absorb timing issues related to network, OS and hardware quality. (Increments of 10ms) (default 50)
+  -to-mumble-buffer int
+     TO_MUMBLE_BUFFER, Jitter buffer from Discord to Mumble to absorb timing issues related to network, OS and hardware quality. (Increments of 10ms) (default 50)
 ```
 
 The bridge can be run with the follow modes:
@@ -67,13 +69,17 @@ In "auto" or "manual" modes, the bridge can be controlled in Discord with the fo
 ```bash
 !DISCORD_COMMAND link
  Commands the bridge to join the Discord channel the user is in and the Mumble server
+
 !DISCORD_COMMAND unlink
  Commands the bridge to leave the Discord channel the user is in and the Mumble server
+
 !DISCORD_COMMAND refresh
  Commands the bridge to unlink, then link again.
+
 !DISCORD_COMMAND auto
  Toggle between manual and auto mode
 ```
+
 ## Setup
 
 ### Creating a Discord Bot
@@ -85,6 +91,7 @@ The guide below provides information on how to setup a Discord bot.
 
 Individual Discord servers need to invite the bot before it can connect.  
 The bot requires the following permissions:
+
 * View Channels
 * See Messages
 * Read Message History
@@ -165,6 +172,16 @@ go build -o mumble-discord-bridge *.go
 make mumble-discord-bridge
 ```
 
+## Jitter Buffer
+
+The bridge implements simple jitter buffers that attempt to compensate for network, OS and hardware related jitter.
+These jitter buffers are configurable in both directions.
+A jitter buffer will slightly the delay the transmission of audio in order to have audio packets buffered for the next time step.
+The Mumble client itself includes a jitter buffer for similar reasons.
+A default jitter of 50ms should be adequate for most scenarios.
+A warning will be logged if short burst or audio are seen.
+A single warning can be ignored multiple warnings in short time spans would suggest the need for a larger jitter buffer.
+
 ## Known Issues
 
 Currently there is an issue opening the discord voice channel.
@@ -187,5 +204,5 @@ Please consider opening an issue to discuss features and ideas.
 
 The project would not have been possible without:
 
-- [gumble](https://github.com/layeh/gumble)
-- [discordgo](https://github.com/bwmarrin/discordgo)
+* [gumble](https://github.com/layeh/gumble)
+* [discordgo](https://github.com/bwmarrin/discordgo)

config.go

@@ -26,6 +26,7 @@ type BridgeConfig struct {
 	MumbleInsecure             bool
 	MumbleCertificate          string
 	MumbleChannel              []string
+	mumbleStartStreamCount     int
 	MumbleDisableText          bool
 	Command                    string
 	GID                        string

discord.go

@@ -102,7 +102,6 @@ func (dd *DiscordDuplex) discordSendPCM(ctx context.Context, wg *sync.WaitGroup,
 		default:
 		}
 
-		// <-ticker.C
 		sleepTick.SleepNextTarget()
 
 		if (len(pcm) > 1 && streaming) || (len(pcm) > dd.Bridge.BridgeConfig.DiscordStartStreamingCount && !streaming) {
@@ -132,14 +131,13 @@ func (dd *DiscordDuplex) discordSendPCM(ctx context.Context, wg *sync.WaitGroup,
 				// The problem delays result in choppy or stuttering sounds, especially when the silence frames are introduced into the opus frames below.
 				// Multiple short cycle delays can result in a Discrod rate limiter being trigger due to of multiple JSON speaking/not-speaking state changes
 				if time.Since(speakingStart).Milliseconds() < 100 {
-					log.Println("Warning: Short Mumble to Discord speaking cycle. Consider increaseing the size of the TO_DISCORD_BUFFER")
+					log.Println("Warning: Short Mumble to Discord speaking cycle. Consider increaseing the size of the to Discord jitter buffer.")
 				}
 
 				// Send silence as suggested by Discord Documentation.
 				// We want to do this after alerting the user of possible short speaking cycles
 				for i := 0; i < 5; i++ {
 					internalSend(opusSilence)
-					// <-ticker.C
 					sleepTick.SleepNextTarget()
 				}
 
@@ -241,11 +239,17 @@ func (dd *DiscordDuplex) discordReceivePCM(ctx context.Context, wg *sync.WaitGro
 }
 
 func (dd *DiscordDuplex) fromDiscordMixer(ctx context.Context, wg *sync.WaitGroup, toMumble chan<- gumble.AudioBuffer) {
+	mumbleSilence := gumble.AudioBuffer{}
+	for i := 3; i < 480; i++ {
+		mumbleSilence = append(mumbleSilence, 0x00)
+	}
+	var speakingStart time.Time
 	sleepTick := SleepCT{
 		d: 10 * time.Millisecond,
 		t: time.Now(),
 	}
 	sendAudio := false
+	toMumbleStreaming := false
 	wg.Add(1)
 
 	for {
@@ -265,9 +269,16 @@ func (dd *DiscordDuplex) fromDiscordMixer(ctx context.Context, wg *sync.WaitGrou
 
 		// Work through each channel
 		for i := range dd.fromDiscordMap {
-			if len(dd.fromDiscordMap[i].pcm) > 0 {
+			bufferLength := len(dd.fromDiscordMap[i].pcm)
+			isStreaming := dd.fromDiscordMap[i].streaming
+			if (bufferLength > 0 && isStreaming) || (bufferLength > dd.Bridge.BridgeConfig.mumbleStartStreamCount && !isStreaming) {
+				if !toMumbleStreaming {
+					speakingStart = time.Now()
+					toMumbleStreaming = true
+				}
 				sendAudio = true
-				if !dd.fromDiscordMap[i].streaming {
+
+				if !isStreaming {
 					x := dd.fromDiscordMap[i]
 					x.streaming = true
 					dd.fromDiscordMap[i] = x
@@ -286,6 +297,22 @@ func (dd *DiscordDuplex) fromDiscordMixer(ctx context.Context, wg *sync.WaitGrou
 
 		dd.discordMutex.Unlock()
 
+		mumbleTimeoutSend := func(outBuf []int16) {
+			timeout := make(chan bool, 1)
+			go func() {
+				time.Sleep(10 * time.Millisecond)
+				timeout <- true
+			}()
+
+			select {
+			case toMumble <- outBuf:
+			case <-timeout:
+				log.Println("toMumble timeout. Dropping packet")
+			}
+		}
+
+		if sendAudio {
+			// Regular send mixed audio
 			outBuf := make([]int16, 480)
 
 			for i := 0; i < len(outBuf); i++ {
@@ -294,13 +321,20 @@ func (dd *DiscordDuplex) fromDiscordMixer(ctx context.Context, wg *sync.WaitGrou
 				}
 			}
 
-		if sendAudio {
-			select {
-			case toMumble <- outBuf:
-			default:
-				log.Println("toMumble buffer full. Dropping packet")
+			mumbleTimeoutSend(outBuf)
+		} else if !sendAudio && toMumbleStreaming {
+			// Send opus silence to mumble
+			// See note above about jitter buffer warning
+			if time.Since(speakingStart).Milliseconds() < 100 {
+				log.Println("Warning: Short Discord to Mumble speaking cycle. Consider increaseing the size of the to Mumble jitter buffer.")
 			}
 
+			for i := 0; i < 5; i++ {
+				mumbleTimeoutSend(mumbleSilence)
+				sleepTick.SleepNextTarget()
+			}
+
+			toMumbleStreaming = false
 		}
 	}
 }

main.go

@@ -42,11 +42,12 @@ func main() {
 	mumbleInsecure := flag.Bool("mumble-insecure", lookupEnvOrBool("MUMBLE_INSECURE", false), " MUMBLE_INSECURE, mumble insecure, optional")
 	mumbleCertificate := flag.String("mumble-certificate", lookupEnvOrString("MUMBLE_CERTIFICATE", ""), "MUMBLE_CERTIFICATE, client certificate to use when connecting to the Mumble server")
 	mumbleChannel := flag.String("mumble-channel", lookupEnvOrString("MUMBLE_CHANNEL", ""), "MUMBLE_CHANNEL, mumble channel to start in, using '/' to separate nested channels, optional")
+	mumbleSendBuffer := flag.Int("to-mumble-buffer", lookupEnvOrInt("TO_MUMBLE_BUFFER", 50), "TO_MUMBLE_BUFFER, Jitter buffer from Discord to Mumble to absorb timing issues related to network, OS and hardware quality. (Increments of 10ms)")
 	mumbleDisableText := flag.Bool("mumble-disable-text", lookupEnvOrBool("MUMBLE_DISABLE_TEXT", false), "MUMBLE_DISABLE_TEXT, disable sending text to mumble, (default false)")
 	discordToken := flag.String("discord-token", lookupEnvOrString("DISCORD_TOKEN", ""), "DISCORD_TOKEN, discord bot token, required")
 	discordGID := flag.String("discord-gid", lookupEnvOrString("DISCORD_GID", ""), "DISCORD_GID, discord gid, required")
 	discordCID := flag.String("discord-cid", lookupEnvOrString("DISCORD_CID", ""), "DISCORD_CID, discord cid, required")
-	discordSendBuffer := flag.Int("to-discord-buffer", lookupEnvOrInt("TO_DISCORD_BUFFER", 50), "TO_DISCORD_BUFFER, Delay buffer from Mumble to Discord to absorb timing issues related to network and hardware quality. (Increments of 10ms)")
+	discordSendBuffer := flag.Int("to-discord-buffer", lookupEnvOrInt("TO_DISCORD_BUFFER", 50), "TO_DISCORD_BUFFER, Jitter buffer from Mumble to Discord to absorb timing issues related to network, OS and hardware quality. (Increments of 10ms)")
 	discordCommand := flag.String("discord-command", lookupEnvOrString("DISCORD_COMMAND", "mumble-discord"), "DISCORD_COMMAND, Discord command string, env alt DISCORD_COMMAND, optional, (defaults mumble-discord)")
 	discordDisableText := flag.Bool("discord-disable-text", lookupEnvOrBool("DISCORD_DISABLE_TEXT", false), "DISCORD_DISABLE_TEXT, disable sending direct messages to discord, (default false)")
 	mode := flag.String("mode", lookupEnvOrString("MODE", "constant"), "MODE, [constant, manual, auto] determine which mode the bridge starts in, (default constant)")
@@ -102,8 +103,15 @@ func main() {
 		*discordSendBuffer = 10
 	}
 
+	if *mumbleSendBuffer < 10 {
+		*mumbleSendBuffer = 10
+	}
+
 	var discordStartStreamingCount int = int(math.Round(float64(*discordSendBuffer) / 10.0))
-	log.Println("Discord Streaming Buffer: ", discordStartStreamingCount*10, " ms")
+	log.Println("To Discord Jitter Buffer: ", discordStartStreamingCount*10, " ms")
+
+	var mumbleStartStreamCount int = int(math.Round(float64(*mumbleSendBuffer) / 10.0))
+	log.Println("To Mumble Jitter Buffer: ", mumbleStartStreamCount*10, " ms")
 
 	// BRIDGE SETUP
 
@@ -114,6 +122,7 @@ func main() {
 			MumbleInsecure:             *mumbleInsecure,
 			MumbleCertificate:          *mumbleCertificate,
 			MumbleChannel:              strings.Split(*mumbleChannel, "/"),
+			mumbleStartStreamCount:     mumbleStartStreamCount,
 			MumbleDisableText:          *mumbleDisableText,
 			Command:                    *discordCommand,
 			GID:                        *discordGID,