diff --git a/handlers.go b/handlers.go
index 69b0cc0..64a6db2 100644
--- a/handlers.go
+++ b/handlers.go
@@ -201,8 +201,8 @@ func (l *Listener) voiceUpdate(s *discordgo.Session, event *discordgo.VoiceState
 				}
 				l.ConnectedLock.Unlock()
 				l.Bridge.DiscordUserCount = count
-				l.UserCountLock.Unlock()
 			}
+			l.UserCountLock.Unlock()
 		}
 
 	}