From f4fd7524488210172deaaa4394d8dd2b79658101 Mon Sep 17 00:00:00 2001 From: opti1337 Date: Thu, 29 Jan 2026 21:34:46 +0600 Subject: [PATCH 01/20] client volumes and dominant client --- server/plugins/socket.ts | 4 ++-- server/socket/webrtc.ts | 37 ++++++++++++++++++++++++++++++++++--- server/types/webrtc.ts | 2 ++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/server/plugins/socket.ts b/server/plugins/socket.ts index e69362f..fff69a0 100644 --- a/server/plugins/socket.ts +++ b/server/plugins/socket.ts @@ -22,8 +22,8 @@ export default fp>( await fastify.io.close() }) - fastify.ready(() => { - registerWebrtcSocket(fastify.io, fastify.mediasoupRouter) + fastify.ready(async () => { + await registerWebrtcSocket(fastify.io, fastify.mediasoupRouter) }) }, { name: 'socket-io', dependencies: ['mediasoup-worker', 'mediasoup-router'] }, diff --git a/server/socket/webrtc.ts b/server/socket/webrtc.ts index 501e10e..6331080 100644 --- a/server/socket/webrtc.ts +++ b/server/socket/webrtc.ts @@ -1,6 +1,7 @@ import type { types } from 'mediasoup' import type { Server as SocketServer } from 'socket.io' import type { + ChadClient, Namespace, SomeSocket, } from '../types/webrtc.ts' @@ -8,9 +9,39 @@ import { consola } from 'consola' import prisma from '../prisma/client.ts' import { socketToClient } from '../utils/socket-to-client.ts' -export default function (io: SocketServer, router: types.Router) { +export default async function (io: SocketServer, router: types.Router) { const namespace: Namespace = io.of('/webrtc') + const audioLevelObserver = await router.createAudioLevelObserver({ + maxEntries: 10, + threshold: -80, + interval: 800, + }) + + const activeSpeakerObserver = await router.createActiveSpeakerObserver() + + audioLevelObserver.on('volumes', async (volumes: types.AudioLevelObserverVolume[]) => { + namespace.emit('speakingPeers', volumes.map(({ producer, volume }) => { + const { clientId } = producer.appData as { clientId: ChadClient['socketId'] } + + return { + clientId, + volume, + } + })) + }) + + audioLevelObserver.on('silence', () => { + namespace.emit('speakingPeers', []) + namespace.emit('activeSpeaker', undefined) + }) + + activeSpeakerObserver.on('dominantspeaker', ({ producer }) => { + const { clientId } = producer.appData as { clientId: ChadClient['socketId'] } + + namespace.emit('activeSpeaker', clientId) + }) + namespace.on('connection', async (socket) => { consola.info('[WebRtc]', 'Client connected', socket.id) @@ -182,8 +213,8 @@ export default function (io: SocketServer, router: types.Router) { ) } - // TODO: Add into the AudioLevelObserver and ActiveSpeakerObserver. - // https://github.com/versatica/mediasoup-demo/blob/v3/server/lib/Room.js#L1276 + await audioLevelObserver.addProducer({ producerId: producer.id }) + await activeSpeakerObserver.addProducer({ producerId: producer.id }) } catch (error) { if (error instanceof Error) { diff --git a/server/types/webrtc.ts b/server/types/webrtc.ts index e2dfd6e..401f3a4 100644 --- a/server/types/webrtc.ts +++ b/server/types/webrtc.ts @@ -118,6 +118,8 @@ export interface ServerToClientEvents { consumerResumed: (arg: { consumerId: string }) => void consumerScore: (arg: { consumerId: string, score: types.ConsumerScore }) => void clientChanged: (clientId: ChadClient['socketId'], client: ChadClient) => void + speakingPeers: (arg: { clientId: ChadClient['socketId'], volume: types.AudioLevelObserverVolume['volume'] }[]) => void + activeSpeaker: (clientId?: ChadClient['socketId']) => void } export interface InterServerEvent {} From aeaea476094a93021d91fa621f3bfd246ab39469 Mon Sep 17 00:00:00 2001 From: opti1337 Date: Thu, 29 Jan 2026 21:40:56 +0600 Subject: [PATCH 02/20] client volumes and dominant client --- server/socket/webrtc.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/socket/webrtc.ts b/server/socket/webrtc.ts index 6331080..358df1c 100644 --- a/server/socket/webrtc.ts +++ b/server/socket/webrtc.ts @@ -22,10 +22,10 @@ export default async function (io: SocketServer, router: types.Router) { audioLevelObserver.on('volumes', async (volumes: types.AudioLevelObserverVolume[]) => { namespace.emit('speakingPeers', volumes.map(({ producer, volume }) => { - const { clientId } = producer.appData as { clientId: ChadClient['socketId'] } + const { socketId } = producer.appData as { socketId: ChadClient['socketId'] } return { - clientId, + clientId: socketId, volume, } })) @@ -37,9 +37,9 @@ export default async function (io: SocketServer, router: types.Router) { }) activeSpeakerObserver.on('dominantspeaker', ({ producer }) => { - const { clientId } = producer.appData as { clientId: ChadClient['socketId'] } + const { socketId } = producer.appData as { socketId: ChadClient['socketId'] } - namespace.emit('activeSpeaker', clientId) + namespace.emit('activeSpeaker', socketId) }) namespace.on('connection', async (socket) => { From fbdceb2e552a0e154f42b9ff616aa57891372b2f Mon Sep 17 00:00:00 2001 From: opti1337 Date: Thu, 29 Jan 2026 21:59:41 +0600 Subject: [PATCH 03/20] client volumes --- client/app/components.d.ts | 1 + client/app/components/ClientRow.vue | 13 +++++++++++-- client/app/composables/use-mediasoup.ts | 9 +++++++++ client/shared/types.ts | 5 +++++ client/src-tauri/tauri.conf.json | 2 +- 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/client/app/components.d.ts b/client/app/components.d.ts index 134d7b9..88eb6f6 100644 --- a/client/app/components.d.ts +++ b/client/app/components.d.ts @@ -17,6 +17,7 @@ declare module 'vue' { PrimeFloatLabel: typeof import('primevue/floatlabel')['default'] PrimeInputText: typeof import('primevue/inputtext')['default'] PrimePassword: typeof import('primevue/password')['default'] + PrimeProgressBar: typeof import('primevue/progressbar')['default'] PrimeScrollPanel: typeof import('primevue/scrollpanel')['default'] PrimeSelect: typeof import('primevue/select')['default'] PrimeSelectButton: typeof import('primevue/selectbutton')['default'] diff --git a/client/app/components/ClientRow.vue b/client/app/components/ClientRow.vue index 620393f..f7c5f4c 100644 --- a/client/app/components/ClientRow.vue +++ b/client/app/components/ClientRow.vue @@ -7,7 +7,12 @@ }" >
- + @@ -61,7 +66,7 @@ const props = defineProps<{ }>() const { outputMuted } = useApp() -const { consumers: allConsumers, micProducer } = useMediasoup() +const { consumers: allConsumers, micProducer, speakingClients } = useMediasoup() const { me } = useClients() const { show } = useFullscreenVideo() @@ -94,6 +99,10 @@ const audioTrack = computed(() => { return audioConsumer.value?.track }) +const speaking = computed(() => { + return speakingClients.value.find(speaker => speaker.clientId === props.client.socketId) +}) + const audioConsumerPaused = ref(false) const inputMuted = computed(() => { diff --git a/client/app/composables/use-mediasoup.ts b/client/app/composables/use-mediasoup.ts index d11e615..8314ee9 100644 --- a/client/app/composables/use-mediasoup.ts +++ b/client/app/composables/use-mediasoup.ts @@ -1,6 +1,8 @@ +import type { SpeakingClient } from '#shared/types' import type { MediaKind, ProducerOptions } from 'mediasoup-client/types' import { createSharedComposable } from '@vueuse/core' import * as mediasoupClient from 'mediasoup-client' +import { shallowRef } from 'vue' import { useDevices } from '~/composables/use-devices' import { usePreferences } from '~/composables/use-preferences' import { useSignaling } from '~/composables/use-signaling' @@ -40,6 +42,8 @@ export const useMediasoup = createSharedComposable(() => { const consumers = shallowRef>(new Map()) const producers = shallowRef>(new Map()) + const speakingClients = shallowRef([]) + watch(signaling.socket, (socket) => { if (!socket) return @@ -227,6 +231,10 @@ export const useMediasoup = createSharedComposable(() => { triggerRef(consumers) }) + + socket.on('speakingPeers', (value: SpeakingClient[]) => { + speakingClients.value = value + }) }, { immediate: true, flush: 'sync' }) async function enableProducer(type: ProducerType, options: ProducerOptions) { @@ -424,6 +432,7 @@ export const useMediasoup = createSharedComposable(() => { init, consumers, producers, + speakingClients, sendTransport, recvTransport, rtpCapabilities, diff --git a/client/shared/types.ts b/client/shared/types.ts index 515bda1..6aec3e8 100644 --- a/client/shared/types.ts +++ b/client/shared/types.ts @@ -8,3 +8,8 @@ export interface ChadClient { } export type UpdatedClient = Omit + +export interface SpeakingClient { + clientId: ChadClient['socketId'] + volume: number +} diff --git a/client/src-tauri/tauri.conf.json b/client/src-tauri/tauri.conf.json index 4b2a4e2..595db15 100644 --- a/client/src-tauri/tauri.conf.json +++ b/client/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "chad", - "version": "0.2.19", + "version": "0.2.20", "identifier": "xyz.koptilnya.chad", "build": { "frontendDist": "../.output/public", From 4c8a0e791c8813a81b5d7cd898535d759b924f0a Mon Sep 17 00:00:00 2001 From: opti1337 Date: Thu, 29 Jan 2026 22:04:40 +0600 Subject: [PATCH 04/20] client volumes --- client/app/pages/preferences.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/app/pages/preferences.vue b/client/app/pages/preferences.vue index d3deb0d..3ab2638 100644 --- a/client/app/pages/preferences.vue +++ b/client/app/pages/preferences.vue @@ -52,11 +52,11 @@
-
- FPS - {{ shareFps }} +
+ Share FPS
- + +